diff --git a/.bumpversion.toml b/.bumpversion.toml
index 6840e4f3138..db4feee7eac 100644
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "9.0.0-beta.16"
+current_version = "10.0.0-beta.3"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -80,6 +80,11 @@ filename = "Cargo.toml"
 search = 'lance-index = {{ version = "={current_version}"'
 replace = 'lance-index = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-index-core = {{ version = "={current_version}"'
+replace = 'lance-index-core = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-io = {{ version = "={current_version}"'
diff --git a/.coderabbit.yaml b/.coderabbit.yaml
new file mode 100644
index 00000000000..9779b0ff8e4
--- /dev/null
+++ b/.coderabbit.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: en-US
+early_access: false
+reviews:
+  profile: quiet
+  high_level_summary: false
+  poem: false
+  review_status: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - WIP
+      - Draft
+    drafts: false
+    base_branches:
+      - main
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 00000000000..0e7363697b0
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,79 @@
+name: Bug report
+description: Report incorrect, unexpected, or crashing behavior.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to file a bug report! Please search
+        [existing issues](https://github.com/lance-format/lance/issues) first to
+        avoid filing a duplicate.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: >-
+        A minimal, self-contained code snippet or sequence of steps that
+        reproduces the problem. The more we can copy-paste and run, the faster
+        we can fix it.
+      placeholder: |
+        1. Write a dataset with `...`
+        2. Scan with filter `...`
+        3. Observe `...`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: false
+  - type: input
+    id: lance-version
+    attributes:
+      label: Lance version
+      description: >-
+        Output of `python -c "import lance; print(lance.__version__)"`, or the
+        `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`.
+      placeholder: "e.g. 0.40.0"
+    validations:
+      required: true
+  - type: dropdown
+    id: language
+    attributes:
+      label: Language binding
+      multiple: true
+      options:
+        - Python
+        - Rust
+        - Java
+        - Other / not sure
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...).
+      placeholder: "e.g. Ubuntu 22.04, x86_64, S3"
+    validations:
+      required: false
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs / traceback
+      description: >-
+        Any relevant log output or stack trace. This will be automatically
+        formatted as code, so no need for backticks.
+      render: shell
+    validations:
+      required: false
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 00000000000..32389799971
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,7 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question / usage help
+    url: https://discord.gg/lance
+    about: >-
+      For questions and general discussion, please ask in the Lance Discord
+      rather than opening an issue.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000000..41d6278fc02
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest a new capability or an improvement to an existing one.
+labels: feature
+---
+
+<!--
+Describe what you'd like Lance to do and why. There's no required format —
+mention the problem or use case, and a rough idea of the solution if you have
+one. Please search existing issues first to avoid duplicates.
+-->
diff --git a/.github/ISSUE_TEMPLATE/performance_issue.md b/.github/ISSUE_TEMPLATE/performance_issue.md
new file mode 100644
index 00000000000..65fcd0be6f4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/performance_issue.md
@@ -0,0 +1,12 @@
+---
+name: Performance issue
+about: Report slow operations, high memory use, or a performance regression.
+labels: performance
+---
+
+<!--
+Describe what's slow or memory-hungry. There's no required format, but concrete
+numbers (timings, memory, dataset shape) and a way to reproduce help a lot. If
+it's a regression, note the last version that performed well. Please search
+existing issues first to avoid duplicates.
+-->
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index a7b7d11c92b..5c5f1aa6113 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -1,45 +1,86 @@
 version: 2
 updates:
+  # Rust — one grouped version-bump PR per directory each week, and security
+  # advisories that land together in a directory batched into a single PR
+  # instead of one PR per advisory.
   - package-ecosystem: "cargo"
-    directory: "/"
+    directories:
+      - "/"
+      - "/python"
+      - "/java/lance-jni"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
       cargo:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      cargo-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
-  - package-ecosystem: "cargo"
+  # Python (uv) — same weekly-grouped + grouped-security treatment.
+  - package-ecosystem: "uv"
     directory: "/python"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      cargo:
+      uv:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      uv-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
+  # Standalone Rust lockfiles for dev/test fixtures. Dependabot security updates
+  # scan these regardless of config, so we group them; open-pull-requests-limit: 0
+  # disables the (unwanted) weekly version bumps for these crates.
   - package-ecosystem: "cargo"
-    directory: "/java/lance-jni"
-    versioning-strategy: lockfile-only
+    directories:
+      - "/memtest"
+      - "/test_data/fri_straddle_pre_6610/datagen"
+    open-pull-requests-limit: 0
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      cargo:
+      cargo-fixtures-security:
+        applies-to: security-updates
         patterns:
           - "*"
 
-  - package-ecosystem: "uv"
-    directory: "/python"
-    versioning-strategy: lockfile-only
+  # Java (Maven). Security updates fire regardless of config; group them and
+  # skip version bumps (limit 0) — we bump Java deps manually.
+  - package-ecosystem: "maven"
+    directory: "/java"
+    open-pull-requests-limit: 0
     schedule:
       interval: "weekly"
       day: "wednesday"
     groups:
-      uv:
+      maven-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Python benchmark scripts (pip / requirements.txt). Group security updates
+  # and skip version bumps (limit 0).
+  - package-ecosystem: "pip"
+    directories:
+      - "/benchmarks/*"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      pip-security:
+        applies-to: security-updates
         patterns:
           - "*"
diff --git a/.github/labeler-issues.yml b/.github/labeler-issues.yml
new file mode 100644
index 00000000000..625b321cd6c
--- /dev/null
+++ b/.github/labeler-issues.yml
@@ -0,0 +1,28 @@
+version: 1
+# Never remove labels a template or a human already applied.
+appendOnly: true
+# Content-based labels for issues, applied by srvaroa/labeler via
+# .github/workflows/issue-labeler.yml. This covers issues that bypass the
+# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`,
+# the REST API, or an agent, which never see the web-UI template chooser.
+#
+# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]",
+# "perf -"), which many people type by hand; a modest body keyword match is a
+# secondary fallback. Rules are OR'd (one label per matching entry), so an
+# unmatched issue gets no label and is left for human / LLM triage rather than
+# guessed at. appendOnly means matches stack harmlessly with template labels.
+labels:
+# --- Primary signal: a leading bug/feature/perf marker in the title ---
+- label: bug
+  title: "(?i)^\\s*[\\[(]?\\s*bug\\b"
+- label: feature
+  title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b"
+- label: performance
+  title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b"
+# --- Secondary fallback: keywords anywhere in the body ---
+- label: bug
+  body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)"
+- label: performance
+  body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)"
+- label: feature
+  body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)"
diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml
new file mode 100644
index 00000000000..36f5a9cdbca
--- /dev/null
+++ b/.github/workflows/issue-labeler.yml
@@ -0,0 +1,29 @@
+name: Issue Labeler
+
+# Applies bug / feature / performance labels to issues based on their title and
+# body. This complements the .github/ISSUE_TEMPLATE forms, which only apply
+# labels for issues opened through the web UI; this catches issues opened via
+# `gh issue create`, the REST API, or an agent, which bypass templates.
+#
+# Issues never originate from a fork, so unlike the PR labelers this uses a
+# plain `issues` trigger (no pull_request_target) with a scoped GITHUB_TOKEN.
+on:
+  issues:
+    types: [opened, edited]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+
+jobs:
+  label:
+    name: Apply issue labels
+    permissions:
+      issues: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          config_path: .github/labeler-issues.yml
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 601f2dd9dfc..cd17c5d0433 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -97,7 +97,7 @@ jobs:
     timeout-minutes: 45
     strategy:
       matrix:
-        python-minor-version: ["10", "13"]
+        python-minor-version: ["10", "13", "14"]
     name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
     runs-on: "ubuntu-24.04-4x"
     defaults:
@@ -124,7 +124,7 @@ jobs:
         with:
           memtest: true
       - name: Upload wheels as artifacts
-        if: ${{ matrix.python-minor-version == '13' }}
+        if: ${{ matrix.python-minor-version == '14' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: linux-wheels
@@ -147,7 +147,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          python-version: 3.13
+          python-version: 3.14
       - name: Download wheels
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
@@ -165,7 +165,7 @@ jobs:
   linux-arm:
     timeout-minutes: 45
     runs-on: ubuntu-24.04-arm64-4x
-    name: Python Linux 3.13 ARM
+    name: Python Linux 3.14 ARM
     defaults:
       run:
         shell: bash
@@ -178,7 +178,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          python-version: 3.13
+          python-version: 3.14
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
@@ -195,7 +195,7 @@ jobs:
       - uses: ./.github/workflows/run_tests
   mac:
     timeout-minutes: 45
-    name: Python macOS 3.13 ARM
+    name: Python macOS 3.14 ARM
     runs-on: "warp-macos-14-arm64-6x"
     defaults:
       run:
@@ -209,7 +209,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
           workspaces: python
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index e3c17671ce3..5e3d8218525 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -42,6 +42,25 @@ jobs:
       - name: Check formatting
         run: cargo fmt -- --check
 
+  package:
+    name: Check crates are publishable
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      # Catches manifest problems that only surface at publish time (missing
+      # readme/license files, path dependencies without a version, excluded
+      # files a crate needs). `--no-verify` skips building each packaged crate,
+      # which the other jobs already cover. Keep the excludes in sync with
+      # .github/workflows/cargo-publish.yml.
+      - name: Package crates
+        run: |
+          cargo package --workspace --no-verify \
+            --exclude lance-arrow-stats \
+            --exclude lance-arrow-scalar \
+            --all-features
+
   rustdoc:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
@@ -85,7 +104,7 @@ jobs:
 
   linux-build:
     runs-on: "ubuntu-24.04-8x"
-    timeout-minutes: 60
+    timeout-minutes: 75
     env:
       # Need up-to-date compilers for kernels
       CC: clang
@@ -96,8 +115,9 @@ jobs:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Setup rust toolchain
         run: |
-          rustup toolchain install nightly
-          rustup default nightly
+          # Temporary mitigation for https://github.com/rust-lang/rust/issues/159261.
+          rustup toolchain install nightly-2026-07-13
+          rustup default nightly-2026-07-13
       - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
@@ -111,7 +131,7 @@ jobs:
       - name: Run tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
-          cargo +nightly llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
+          cargo +nightly-2026-07-13 llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         with:
@@ -253,6 +273,96 @@ jobs:
       - name: Check benchmarks
         run: cargo check --profile ci --benches
 
+  qemu-pre-haswell:
+    # Verifies that lance-linalg's runtime SIMD dispatch still works
+    # correctly when the binary is built with the lower x86-64-v2 baseline
+    # (the legacy build path documented in CONTRIBUTING.md). Emulates a
+    # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA
+    # instructions that leak past the runtime dispatch.
+    #
+    # The published-wheel default baseline (`target-cpu=haswell`) is set in
+    # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to
+    # exercise the legacy path without affecting any other build.
+    name: pre-Haswell SIGILL check (qemu Nehalem)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      CC: clang
+      CXX: clang++
+      RUSTFLAGS: "-C target-cpu=x86-64-v2"
+      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem"
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Setup rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Restore QEMU 8.2.10
+        id: qemu-cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/lance-qemu/8.2.10/qemu-x86_64
+          key: qemu-user-8.2.10-x86_64-linux-user-${{ runner.os }}-${{ runner.arch }}-v1
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install QEMU build dependencies
+        if: steps.qemu-cache.outputs.cache-hit != 'true'
+        run: |
+          sudo apt install -y \
+            ninja-build \
+            pkg-config \
+            libglib2.0-dev \
+            python3-venv \
+            curl \
+            xz-utils
+      - name: Build QEMU 8.2.10
+        if: steps.qemu-cache.outputs.cache-hit != 'true'
+        run: |
+          QEMU_VERSION=8.2.10
+          QEMU_ARCHIVE="qemu-${QEMU_VERSION}.tar.xz"
+          QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/${QEMU_VERSION}"
+          QEMU_SOURCE_DIR="${RUNNER_TEMP}/qemu-${QEMU_VERSION}"
+
+          # Ubuntu 24.04 ships QEMU 8.2.2, which is affected by
+          # https://gitlab.com/qemu-project/qemu/-/issues/2170.
+          curl --fail --location --silent --show-error \
+            "https://download.qemu.org/${QEMU_ARCHIVE}" \
+            --output "${RUNNER_TEMP}/${QEMU_ARCHIVE}"
+          echo "37b4a643da8ed6015eef35f5d7f06e7259d9c95359965a0a98e9667c621ab2bb  ${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
+            | sha256sum --check -
+          tar --extract \
+            --file "${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
+            --directory "${RUNNER_TEMP}"
+          mkdir "${QEMU_SOURCE_DIR}/build"
+          (
+            cd "${QEMU_SOURCE_DIR}/build"
+            ../configure --target-list=x86_64-linux-user --disable-docs
+            ninja qemu-x86_64
+          )
+          mkdir -p "${QEMU_BIN_DIR}"
+          install -m 0755 \
+            "${QEMU_SOURCE_DIR}/build/qemu-x86_64" \
+            "${QEMU_BIN_DIR}/qemu-x86_64"
+      - name: Add QEMU to PATH
+        run: |
+          QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/8.2.10"
+          echo "${QEMU_BIN_DIR}" >> "${GITHUB_PATH}"
+          QEMU_VERSION_OUTPUT="$("${QEMU_BIN_DIR}/qemu-x86_64" --version)"
+          echo "${QEMU_VERSION_OUTPUT}"
+          if [[ "${QEMU_VERSION_OUTPUT}" != *"version 8.2.10"* ]]; then
+            echo "Expected QEMU 8.2.10, got: ${QEMU_VERSION_OUTPUT}" >&2
+            exit 1
+          fi
+      - name: Build lance-linalg lib tests in release mode
+        run: |
+          cargo test --release -p lance-linalg --lib --no-run
+      - name: Run lance-linalg lib tests under qemu Nehalem
+        run: |
+          cargo test --release -p lance-linalg --lib
+
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}
diff --git a/.gitignore b/.gitignore
index dcc9c5089ff..8143d39eef6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -105,6 +105,7 @@ test_data/venv
 
 **/*.profraw
 *.lance
+!test_data/**/*.lance
 
 # Pytest benchmarks
 .benchmarks/
diff --git a/AGENTS.md b/AGENTS.md
index ee9b3b07e9d..78ff6331d34 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -4,31 +4,11 @@ Lance is a modern columnar data format optimized for ML workflows and datasets,
 
 Also see directory-specific guidelines: [rust/](rust/AGENTS.md) | [python/](python/AGENTS.md) | [java/](java/AGENTS.md) | [protos/](protos/AGENTS.md) | [docs/src/format/](docs/src/format/AGENTS.md)
 
-## Architecture
-
-Rust workspace with Python and Java bindings:
-
-- `rust/lance/` - Main library implementing the columnar format
-- `rust/lance-core/` - Core types, traits, and utilities
-- `rust/lance-arrow/` - Apache Arrow integration layer
-- `rust/lance-encoding/` - Data encoding and compression algorithms
-- `rust/lance-file/` - File format reading/writing
-- `rust/lance-index/` - Vector and scalar indexing
-- `rust/lance-io/` - I/O operations and object store integration
-- `rust/lance-linalg/` - Linear algebra for vector search
-- `rust/lance-table/` - Table format and operations
-- `rust/lance-geo/` - Geospatial data support
-- `rust/lance-datagen/` - Data generation for tests and benchmarks
-- `rust/lance-namespace/` / `rust/lance-namespace-impls/` - Namespace/catalog interfaces
-- `rust/lance-test-macros/` / `rust/lance-testing/` - Test infrastructure
-- `rust/lance-tools/` - CLI and developer tooling
-- `rust/examples/` - Sample binaries and demonstrations
-- `rust/compression/bitpacking/` / `rust/compression/fsst/` - Compression codecs
-- `rust/lance-datafusion/` - DataFusion integration (built separately)
-- `python/` - Python bindings (PyO3/maturin)
-- `java/` - Java bindings (JNI)
-
-Key technical traits: async-first (tokio), Arrow-native, versioned writes with manifest tracking, custom ML-optimized encodings, unified object store interface (local/S3/Azure/GCS).
+## File Format Stability and Compatibility
+
+- Treat every file format marked stable as a durable compatibility contract. All changes to a stable format must preserve both backward and forward compatibility.
+- Treat every file format marked unstable as disposable. It may change freely; do not add compatibility code, migrations, fallbacks, or tests for files written by earlier unstable revisions.
+- Evaluate compatibility against the latest released stable version while continuing to honor all stable format contracts. Changes that exist only on the current branch or `main` are not compatibility constraints; do not compromise a cleaner or more complete design to preserve those intermediate states.
 
 ## Development Commands
 
@@ -134,6 +114,11 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p
 - Indent content under MkDocs admonition directives (`!!! note`, etc.) with 4 spaces.
 - Proofread comments and docs for typos before committing.
 
+## Filing Issues
+
+- When opening an issue with `gh issue create` or the API, classify it and pass the matching label: `--label bug`, `--label feature`, or `--label performance`. These paths bypass the `.github/ISSUE_TEMPLATE` forms, so the label is not applied automatically.
+- Prefix the title to match, e.g. `bug: ...`, `feature: ...`, or `perf: ...`. A content-based labeler (`.github/workflows/issue-labeler.yml`) uses this as a fallback signal, but an explicit `--label` is the reliable path.
+
 ## Pull Requests
 
 - PR titles must follow the Conventional Commits specification because `.github/workflows/pr-title.yml` validates the PR title and body with commitlint. Use prefixes like `feat:`, `fix:`, `docs:`, `perf:`, `ci:`, `test:`, `build:`, `style:`, or `chore:`; add a scope when useful.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8f3ec285f31..3940ed7b683 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -25,6 +25,16 @@ Currently Lance is implemented in Rust and comes with a Python wrapper. So you'l
     a. Install pre-commit: https://pre-commit.com/#install
     b. Run `pre-commit install` in the root of the repo
 
+## Building for legacy x86_64 hosts (pre-Haswell)
+
+The default workspace build targets `haswell` (AVX2 + FMA + F16C), matching the published wheels. To build a binary that runs on pre-Haswell silicon (Sandy Bridge / Ivy Bridge / Westmere on Intel, Bulldozer / Piledriver / Steamroller on AMD — i.e. CPUs without AVX2), set the baseline yourself at build time:
+
+```sh
+RUSTFLAGS="-C target-cpu=x86-64-v2" cargo build --release
+```
+
+Runtime SIMD dispatch in `lance-linalg::distance` will then pick the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) based on the host. From Python, use `lance.simd_info()` to verify which tier was selected.
+
 ## Sample Workflow
 
 1. Fork the repo
diff --git a/Cargo.lock b/Cargo.lock
index f537f382479..791030cff2d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -205,9 +205,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -219,9 +219,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -238,9 +238,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
 dependencies = [
  "bytes",
  "half",
@@ -250,9 +250,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -287,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -300,9 +300,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -341,9 +341,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -354,9 +354,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -367,9 +367,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
 dependencies = [
  "bitflags 2.13.0",
  "serde_core",
@@ -378,9 +378,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -460,18 +460,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -1225,7 +1225,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1236,9 +1236,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
 
 [[package]]
 name = "byteorder"
@@ -1248,9 +1262,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1279,9 +1293,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.65"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1390,9 +1404,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1400,9 +1414,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1412,14 +1426,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -1734,18 +1748,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1908,7 +1922,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1921,7 +1935,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1943,7 +1957,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
  "darling_core 0.20.11",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1954,7 +1968,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
  "darling_core 0.23.0",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1973,14 +1987,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -2007,12 +2020,11 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -2022,9 +2034,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2047,9 +2059,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2070,32 +2082,33 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
- "paste",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
 dependencies = [
  "futures",
  "log",
@@ -2104,9 +2117,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2126,16 +2139,17 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.4",
+ "parking_lot",
+ "rand 0.9.5",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -2157,9 +2171,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2180,9 +2194,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2197,27 +2211,25 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -2226,18 +2238,19 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -2248,29 +2261,27 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2285,26 +2296,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -2314,19 +2324,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -2335,9 +2344,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2351,34 +2360,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2389,14 +2398,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2404,20 +2412,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
 dependencies = [
  "arrow",
  "chrono",
@@ -2434,11 +2442,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2446,20 +2453,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2472,26 +2478,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2507,12 +2513,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2527,7 +2534,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2539,9 +2546,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2550,15 +2557,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2570,9 +2576,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2588,9 +2594,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2693,7 +2699,7 @@ dependencies = [
  "darling 0.20.11",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2713,7 +2719,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
 dependencies = [
  "derive_builder_core 0.20.2",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2775,7 +2781,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2860,6 +2866,12 @@ dependencies = [
  "encoding_rs",
 ]
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "env_filter"
 version = "2.0.0"
@@ -2900,7 +2912,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3076,10 +3088,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
- "rand 0.9.4",
+ "rand 0.9.5",
  "test-log",
  "tokio",
 ]
@@ -3155,7 +3167,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3312,12 +3324,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -3413,7 +3426,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3446,7 +3459,7 @@ dependencies = [
  "hostname",
  "prost",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "serde",
  "thiserror 2.0.18",
@@ -3483,6 +3496,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -3535,6 +3549,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3584,7 +3603,7 @@ dependencies = [
  "log",
  "native-tls",
  "num_cpus",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "serde",
  "serde_json",
@@ -3714,9 +3733,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
@@ -4198,18 +4217,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82"
+checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.10.1"
+version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9"
+checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d"
 dependencies = [
  "bytecount",
  "cedarwood",
@@ -4244,7 +4263,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4289,7 +4308,7 @@ dependencies = [
  "quote",
  "rustc_version",
  "simd_cesu8",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4308,7 +4327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
 dependencies = [
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4346,7 +4365,7 @@ dependencies = [
  "nom 8.0.0",
  "num-traits",
  "ordered-float 5.3.0",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "serde_json",
  "zmij",
@@ -4380,7 +4399,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "all_asserts",
  "approx",
@@ -4457,7 +4476,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
  "reqwest 0.12.28",
  "roaring",
@@ -4483,7 +4502,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4492,13 +4511,14 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -4531,7 +4551,7 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrayref",
  "bitpacking",
@@ -4542,7 +4562,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4566,7 +4586,7 @@ dependencies = [
  "pin-project",
  "proptest",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "roaring",
  "rstest",
  "serde_json",
@@ -4582,7 +4602,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4615,7 +4635,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4627,23 +4647,23 @@ dependencies = [
  "half",
  "hex",
  "lance-testing",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rand_xoshiro",
 ]
 
 [[package]]
 name = "lance-derive"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4674,7 +4694,7 @@ dependencies = [
  "prost",
  "prost-build",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_xoshiro",
  "rstest",
  "serial_test",
@@ -4688,7 +4708,7 @@ dependencies = [
 
 [[package]]
 name = "lance-examples"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "all_asserts",
  "arrow",
@@ -4706,7 +4726,7 @@ dependencies = [
  "lance-linalg",
  "object_store",
  "parquet",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "tokenizers",
  "tokio",
@@ -4714,7 +4734,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4753,7 +4773,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4767,13 +4787,14 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "approx",
  "arc-swap",
  "arrow",
  "arrow-arith",
  "arrow-array",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -4810,6 +4831,7 @@ dependencies = [
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
@@ -4825,7 +4847,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rangemap",
  "rayon",
@@ -4842,9 +4864,31 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "10.0.0-beta.3"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4863,6 +4907,7 @@ dependencies = [
  "chrono",
  "criterion",
  "futures",
+ "goosefs-sdk",
  "http 1.4.2",
  "io-uring",
  "lance-arrow",
@@ -4870,6 +4915,8 @@ dependencies = [
  "lance-namespace",
  "lance-testing",
  "log",
+ "metrics",
+ "metrics-util",
  "mock_instant",
  "mockall",
  "moka",
@@ -4879,7 +4926,7 @@ dependencies = [
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rstest",
  "serde",
  "tempfile",
@@ -4892,7 +4939,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "approx",
  "arrow-array",
@@ -4906,13 +4953,14 @@ dependencies = [
  "lance-testing",
  "num-traits",
  "proptest",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
+ "rstest",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4924,7 +4972,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-datafusion"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4940,7 +4988,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4968,8 +5016,8 @@ dependencies = [
  "log",
  "object_store",
  "opendal",
- "quick-xml 0.38.4",
- "rand 0.9.4",
+ "quick-xml 0.40.1",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "ring",
  "roaring",
@@ -5004,7 +5052,7 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5022,7 +5070,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5052,7 +5100,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rangemap",
  "roaring",
  "rstest",
@@ -5068,16 +5116,16 @@ dependencies = [
 
 [[package]]
 name = "lance-test-macros"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "lance-testing"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5085,12 +5133,12 @@ dependencies = [
  "lance-arrow",
  "num-traits",
  "pprof",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
 name = "lance-tokenizer"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",
@@ -5103,7 +5151,7 @@ dependencies = [
 
 [[package]]
 name = "lance-tools"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "clap",
  "lance-core",
@@ -5182,9 +5230,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libm"
@@ -5458,6 +5506,36 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.24.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
+dependencies = [
+ "portable-atomic",
+ "rapidhash",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
+dependencies = [
+ "aho-corasick",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.5",
+ "indexmap 2.14.0",
+ "metrics",
+ "ordered-float 4.6.0",
+ "quanta",
+ "radix_trie",
+ "rand 0.9.5",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -5530,7 +5608,7 @@ dependencies = [
  "cfg-if 1.0.4",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5572,7 +5650,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5604,7 +5682,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5639,6 +5717,15 @@ dependencies = [
  "rawpointer",
 ]
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -5797,7 +5884,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6237,7 +6324,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6273,6 +6360,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-float"
 version = "5.3.0"
@@ -6348,9 +6444,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -6557,7 +6653,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6750,7 +6846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6764,9 +6860,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.106"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
 dependencies = [
  "unicode-ident",
 ]
@@ -6781,7 +6877,7 @@ dependencies = [
  "bit-vec",
  "bitflags 2.13.0",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_chacha 0.9.0",
  "rand_xorshift",
  "regex-syntax",
@@ -6815,7 +6911,7 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn 2.0.118",
+ "syn 2.0.119",
  "tempfile",
 ]
 
@@ -6829,7 +6925,7 @@ dependencies = [
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6867,7 +6963,22 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
 ]
 
 [[package]]
@@ -6885,15 +6996,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -6944,7 +7046,7 @@ dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.9.5",
  "ring",
  "rustc-hash",
  "rustls",
@@ -6972,9 +7074,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.46"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
 dependencies = [
  "proc-macro2",
 ]
@@ -6997,6 +7099,16 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rancor"
 version = "0.1.1"
@@ -7019,9 +7131,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -7089,7 +7201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -7116,6 +7228,24 @@ version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
 
+[[package]]
+name = "rapidhash"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "11.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+dependencies = [
+ "bitflags 2.13.0",
+]
+
 [[package]]
 name = "rawpointer"
 version = "0.2.1"
@@ -7199,7 +7329,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7557,7 +7687,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7633,7 +7763,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.118",
+ "syn 2.0.119",
  "unicode-ident",
 ]
 
@@ -7665,9 +7795,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -7870,7 +8000,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7937,9 +8067,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -7947,22 +8077,22 @@ dependencies = [
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -7973,15 +8103,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -8008,7 +8139,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8020,7 +8151,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8064,7 +8195,7 @@ dependencies = [
  "darling 0.23.0",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8115,7 +8246,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8246,6 +8377,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -8260,23 +8397,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522"
+checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7"
 dependencies = [
  "snafu-derive",
 ]
 
 [[package]]
 name = "snafu-derive"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda"
+checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8351,9 +8488,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -8367,7 +8504,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8459,7 +8596,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8471,16 +8608,17 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -8494,7 +8632,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.118",
+ "syn 2.0.119",
  "typify",
  "walkdir",
 ]
@@ -8547,9 +8685,20 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.118"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8573,7 +8722,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8661,7 +8810,7 @@ checksum = "c26ef8b00e4d382e59f6a8ddb3cd790b3a5bb29f21a358a9a69ea2f29f13f27b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8670,7 +8819,7 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "944ad38adcbb71eaa682c56bceeb079e4ca82b4b3edc2a0fde5cb297b77dac8d"
 dependencies = [
- "syn 2.0.118",
+ "syn 2.0.119",
  "test-log-core",
 ]
 
@@ -8700,7 +8849,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8711,7 +8860,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8854,9 +9003,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
 dependencies = [
  "bytes",
  "libc",
@@ -8877,7 +9026,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8913,9 +9062,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -8925,13 +9074,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
+ "libc",
  "pin-project-lite",
  "tokio",
 ]
@@ -9109,7 +9259,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -9203,11 +9353,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "twox-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
 dependencies = [
- "rand 0.9.4",
+ "rand 0.10.1",
 ]
 
 [[package]]
@@ -9247,7 +9397,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.118",
+ "syn 2.0.119",
  "thiserror 2.0.18",
  "unicode-ident",
 ]
@@ -9265,7 +9415,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.118",
+ "syn 2.0.119",
  "typify-impl",
 ]
 
@@ -9399,9 +9549,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.4"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
  "getrandom 0.4.3",
  "js-sys",
@@ -9536,7 +9686,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
  "wasm-bindgen-shared",
 ]
 
@@ -9719,7 +9869,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -9730,7 +9880,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -10185,9 +10335,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.16"
+version = "0.8.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
+checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
 
 [[package]]
 name = "yansi"
@@ -10214,7 +10364,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
  "synstructure",
 ]
 
@@ -10235,7 +10385,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -10255,7 +10405,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
  "synstructure",
 ]
 
@@ -10297,7 +10447,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 2.0.119",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 486aa1e56a2..b169a252365 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
     "rust/lance-file",
     "rust/lance-geo",
     "rust/lance-index",
+    "rust/lance-index-core",
     "rust/lance-io",
     "rust/lance-linalg",
     "rust/lance-namespace",
@@ -32,7 +33,7 @@ resolver = "3"
 
 
 [workspace.package]
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -57,27 +58,28 @@ rust-version = "1.91.0"
 [workspace.dependencies]
 arc-swap = "1.7"
 libc = "0.2.176"
-lance = { version = "=9.0.0-beta.16", path = "./rust/lance", default-features = false }
-lance-arrow = { version = "=9.0.0-beta.16", path = "./rust/lance-arrow" }
-lance-core = { version = "=9.0.0-beta.16", path = "./rust/lance-core" }
-lance-datafusion = { version = "=9.0.0-beta.16", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=9.0.0-beta.16", path = "./rust/lance-datagen" }
-lance-derive = { version = "=9.0.0-beta.16", path = "./rust/lance-derive" }
-lance-encoding = { version = "=9.0.0-beta.16", path = "./rust/lance-encoding" }
-lance-file = { version = "=9.0.0-beta.16", path = "./rust/lance-file" }
-lance-geo = { version = "=9.0.0-beta.16", path = "./rust/lance-geo" }
-lance-index = { version = "=9.0.0-beta.16", path = "./rust/lance-index" }
-lance-io = { version = "=9.0.0-beta.16", path = "./rust/lance-io", default-features = false }
-lance-linalg = { version = "=9.0.0-beta.16", path = "./rust/lance-linalg" }
-lance-namespace = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace" }
-lance-namespace-impls = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace-impls" }
+lance = { version = "=10.0.0-beta.3", path = "./rust/lance", default-features = false }
+lance-arrow = { version = "=10.0.0-beta.3", path = "./rust/lance-arrow" }
+lance-core = { version = "=10.0.0-beta.3", path = "./rust/lance-core" }
+lance-datafusion = { version = "=10.0.0-beta.3", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=10.0.0-beta.3", path = "./rust/lance-datagen" }
+lance-derive = { version = "=10.0.0-beta.3", path = "./rust/lance-derive" }
+lance-encoding = { version = "=10.0.0-beta.3", path = "./rust/lance-encoding" }
+lance-file = { version = "=10.0.0-beta.3", path = "./rust/lance-file" }
+lance-geo = { version = "=10.0.0-beta.3", path = "./rust/lance-geo" }
+lance-index = { version = "=10.0.0-beta.3", path = "./rust/lance-index" }
+lance-index-core = { version = "=10.0.0-beta.3", path = "./rust/lance-index-core" }
+lance-io = { version = "=10.0.0-beta.3", path = "./rust/lance-io", default-features = false }
+lance-linalg = { version = "=10.0.0-beta.3", path = "./rust/lance-linalg" }
+lance-namespace = { version = "=10.0.0-beta.3", path = "./rust/lance-namespace" }
+lance-namespace-impls = { version = "=10.0.0-beta.3", path = "./rust/lance-namespace-impls" }
 lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" }
 lance-namespace-reqwest-client = "0.8.6"
-lance-select = { version = "=9.0.0-beta.16", path = "./rust/lance-select" }
-lance-tokenizer = { version = "=9.0.0-beta.16", path = "./rust/lance-tokenizer" }
-lance-table = { version = "=9.0.0-beta.16", path = "./rust/lance-table" }
-lance-test-macros = { version = "=9.0.0-beta.16", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=9.0.0-beta.16", path = "./rust/lance-testing" }
+lance-select = { version = "=10.0.0-beta.3", path = "./rust/lance-select" }
+lance-tokenizer = { version = "=10.0.0-beta.3", path = "./rust/lance-tokenizer" }
+lance-table = { version = "=10.0.0-beta.3", path = "./rust/lance-table" }
+lance-test-macros = { version = "=10.0.0-beta.3", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=10.0.0-beta.3", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false, features = ["prettyprint"] }
@@ -103,10 +105,14 @@ aws-sdk-s3 = { version = "1.38.0", default-features = false }
 half = { "version" = "2.1", default-features = false, features = [
     "num-traits",
     "std",
+    "bytemuck",
 ] }
-lance-bitpacking = { version = "=9.0.0-beta.16", path = "./rust/compression/bitpacking" }
+lance-bitpacking = { version = "=10.0.0-beta.3", path = "./rust/compression/bitpacking" }
 bitpacking = "0.9"
 bitvec = "1"
+bytemuck = { version = "1", default-features = false, features = [
+    "extern_crate_alloc",
+] }
 bytes = "1.11.1"
 byteorder = "1.5"
 clap = { version = "4", features = ["derive"] }
@@ -122,7 +128,7 @@ criterion = { version = "0.8.2", features = [
 ] }
 crossbeam-queue = "0.3"
 crossbeam-skiplist = "0.1"
-datafusion = { version = "53.0.0", default-features = false, features = [
+datafusion = { version = "54.0.0", default-features = false, features = [
     "crypto_expressions",
     "datetime_expressions",
     "encoding_expressions",
@@ -132,24 +138,25 @@ datafusion = { version = "53.0.0", default-features = false, features = [
     "string_expressions",
     "unicode_expressions",
 ] }
-datafusion-common = "53.0.0"
-datafusion-functions = { version = "53.0.0", default-features = false, features = ["regex_expressions"] }
-datafusion-sql = "53.0.0"
-datafusion-expr = "53.0.0"
-datafusion-ffi = "53.0.0"
-datafusion-physical-expr = "53.0.0"
-datafusion-physical-plan = "53.0.0"
-datafusion-substrait = { version = "53.0.0", default-features = false }
+datafusion-common = "54.0.0"
+datafusion-functions = { version = "54.0.0", default-features = false, features = ["regex_expressions"] }
+datafusion-sql = "54.0.0"
+datafusion-expr = "54.0.0"
+datafusion-ffi = "54.0.0"
+datafusion-physical-expr = "54.0.0"
+datafusion-physical-plan = "54.0.0"
+datafusion-substrait = { version = "54.0.0", default-features = false }
 dirs = "6.0.0"
 either = "1.0"
 fst = { version = "0.4.7", features = ["levenshtein"] }
-fsst = { version = "=9.0.0-beta.16", path = "./rust/compression/fsst" }
+fsst = { version = "=10.0.0-beta.3", path = "./rust/compression/fsst" }
 futures = "0.3"
 geoarrow-array = "0.8"
 geoarrow-schema = "0.8"
-geodatafusion = "0.4.0"
+geodatafusion = "0.5.0"
 geo-traits = "0.3.0"
 geo-types = "0.7.16"
+goosefs-sdk = "=0.1.5"
 http = "1.1.0"
 humantime = "2.2.0"
 hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }
@@ -160,6 +167,8 @@ jieba-rs = { version = "0.10.0", default-features = false }
 jsonb = { version = "0.5.3", default-features = false, features = ["databend"] }
 libm = "0.2.15"
 log = "0.4"
+metrics = { version = "0.24" }
+metrics-util = { version = "0.19" }
 mockall = { version = "0.14.0" }
 mock_instant = { version = "0.6.0" }
 moka = { version = "0.12", features = ["future", "sync"] }
diff --git a/README.md b/README.md
index 886fd70425e..08716c84ead 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,17 @@ For more details, see the full [Lance format specification](https://lance.org/fo
 > [!TIP]
 > Lance is in active development and we welcome contributions. Please see our [contributing guide](https://lance.org/community/contributing/) for more information.
 
+## File format stability
+
+Lance releases frequently because the SDKs, integrations, and performance work are moving quickly. This does not mean the Lance file format changes incompatibly in every release. The Lance file format is identified by the `data_storage_version` stored in each dataset, and stable storage versions are a long-term compatibility contract.
+
+* Once a dataset is written with a stable `data_storage_version`, future Lance releases will continue to support reading that storage version.
+* SDK and API compatibility is separate from file format compatibility. SDK/API changes follow semantic versioning and are documented in the [migration guide](https://lance.org/guide/migration/).
+* Older Lance releases may not understand file format versions introduced later. If you run mixed Lance versions, pin `data_storage_version` for deterministic writes.
+* The `next` file format alias is unstable and should only be used for experimentation, never for production data.
+
+For production, write data with a stable `data_storage_version`. See the [format versioning guide](https://lance.org/format/file/versioning/) for the current compatibility matrix.
+
 ## Quick Start
 
 **Installation**
diff --git a/docs/README.md b/docs/README.md
index 80092b157c7..8cb33f86387 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,7 @@
 # Lance Documentation
 
-This directory contains the documentation for Lance, built with MkDocs and Material theme.
+This directory contains the documentation for Lance, built with MkDocs and a
+custom theme (`theme/`) implementing the Lance Docs design.
 
 ## Getting Started with uv
 
@@ -62,5 +63,18 @@ uv sync --upgrade
 ## Project Structure
 
 - `src/` - Source markdown files for documentation
+- `theme/` - Custom MkDocs theme implementing the Lance Docs design
+  (templates in `*.html`, styles/behaviour in `theme/assets/`)
 - `mkdocs.yml` - MkDocs configuration
 - `pyproject.toml` - Python project configuration (uv compatible)
+
+## Theme Notes
+
+- Light/dark mode follows `prefers-color-scheme`, is toggleable from the
+  header, and persists in `localStorage` (`ld-theme`).
+- The GitHub star count is fetched from the public GitHub API and cached in
+  `localStorage` for an hour; the button degrades to a plain link offline.
+- Search is a lightweight overlay (`/` or `Cmd/Ctrl+K`) over the standard
+  `search` plugin index — no external search dependencies.
+- Mermaid diagrams render client-side; the library is loaded from a CDN only
+  on pages that contain a diagram.
diff --git a/docs/clean-full-website.sh b/docs/clean-full-website.sh
index db8013cd744..2ebe0b31b87 100755
--- a/docs/clean-full-website.sh
+++ b/docs/clean-full-website.sh
@@ -39,7 +39,7 @@ nav:
   - Apache DataFusion: datafusion.md
   - PostgreSQL: https://github.com/lancedb/pglance
   - PyTorch: pytorch.md
-  - Tensorflow: tensorflow.md
+  - TensorFlow: tensorflow.md
 EOF
 
 mkdir -p "$docs_src/format/catalog/dir"
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 8144a42e5f5..de75e65f5d9 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -6,37 +6,13 @@ docs_dir: src
 repo_name: lance-format/lance
 repo_url: https://github.com/lance-format/lance
 
+# Custom theme implementing the "Lance Docs" design (see theme/).
 theme:
-  name: material
-  custom_dir: overrides
-  logo: logo/white.png
-  favicon: logo/logo.png
-  palette:
-    - scheme: default
-      primary: custom
-      accent: custom
-      toggle:
-        icon: material/brightness-7
-        name: Switch to dark mode
-    - scheme: slate
-      primary: custom
-      accent: custom
-      toggle:
-        icon: material/brightness-4
-        name: Switch to light mode
-  features:
-    - navigation.tabs
-    - navigation.sections
-    - navigation.instant
-    - navigation.indexes
-    - navigation.tracking
-    - navigation.top
-    - search.highlight
-    - search.share
-    - content.code.copy
-    - content.code.annotate
-  icon:
-    repo: fontawesome/brands/github
+  name: null
+  custom_dir: theme
+  locale: en
+  static_templates:
+    - 404.html
 
 markdown_extensions:
   - admonition
@@ -51,9 +27,16 @@ markdown_extensions:
       line_spans: __span
       pygments_lang_class: true
   - pymdownx.inlinehilite
-  - pymdownx.snippets
+  - pymdownx.snippets:
+      # Allow snippets to pull in files from the repo root (e.g. shared docs
+      # kept alongside Rust source). Paths are relative to the docs/ dir.
+      base_path:
+        - .
+        - ..
   - pymdownx.tabbed:
       alternate_style: true
+  # Autolink bare URLs (e.g. repository tables on the community pages).
+  - pymdownx.magiclink
   - attr_list
   - md_in_html
   - tables
@@ -66,18 +49,4 @@ plugins:
   - mkdocs_protobuf:
       proto_dir: ../protos
 
-extra:
-  generator: false
-  social:
-    - icon: fontawesome/brands/github
-      link: https://github.com/lance-format/lance
-    - icon: fontawesome/brands/discord
-      link: https://discord.gg/lance
-
 copyright: © 2025 Lance Format. All rights reserved.
-
-extra_css:
-  - assets/stylesheets/home.css
-extra_javascript:
-  - assets/javascripts/nav-expand.js
-
diff --git a/docs/overrides/home.html b/docs/overrides/home.html
deleted file mode 100644
index 26eb288aee6..00000000000
--- a/docs/overrides/home.html
+++ /dev/null
@@ -1,295 +0,0 @@
-{% extends "main.html" %}
-
-{% block tabs %}
-  {{ super() }}
-
-  <style>
-    /* Prevent horizontal overflow */
-    body {
-      overflow-x: hidden;
-    }
-
-    /* Hide main content for home page */
-    .md-content {
-      display: none;
-    }
-
-    /* Hide table of contents */
-    @media screen and (min-width: 60em) {
-      .md-sidebar--secondary {
-        display: none;
-      }
-    }
-
-    /* Hide navigation */
-    @media screen and (min-width: 76.25em) {
-      .md-sidebar--primary {
-        display: none;
-      }
-    }
-
-    /* Make header static */
-    .md-header {
-      position: initial;
-    }
-
-    .md-main__inner {
-      margin: 0;
-    }
-
-    /* Style Learn More and View Integrations links */
-    .lance-feature-section .md-button,
-    .lance-intro-section .md-button:not(.md-button--primary) {
-      background-color: transparent;
-      color: inherit;
-      text-decoration: none;
-      border: none;
-      box-shadow: none;
-    }
-
-    .lance-feature-section .md-button:hover,
-    .lance-intro-section .md-button:not(.md-button--primary):hover {
-      background-color: transparent;
-      color: #625EFF;
-      text-decoration: none;
-    }
-  </style>
-
-  <script>
-    // Fix palette toggle jumping to bottom of page
-    (function() {
-      // Store scroll position
-      let savedScrollY = 0;
-      let isTogglingPalette = false;
-
-      // Prevent hash-based scrolling for palette changes
-      if ('scrollRestoration' in history) {
-        history.scrollRestoration = 'manual';
-      }
-
-      // Intercept scroll attempts during palette toggle
-      const preventScroll = function() {
-        if (isTogglingPalette) {
-          window.scrollTo(0, savedScrollY);
-        }
-      };
-
-      // Save scroll position and intercept clicks on palette toggle
-      document.addEventListener('click', function(e) {
-        const paletteLabel = e.target.closest('[for^="__palette"]');
-        if (paletteLabel) {
-          savedScrollY = window.scrollY;
-          isTogglingPalette = true;
-
-          // Prevent any scrolling for a brief moment
-          window.addEventListener('scroll', preventScroll, { passive: false });
-
-          setTimeout(function() {
-            isTogglingPalette = false;
-            window.removeEventListener('scroll', preventScroll);
-          }, 100);
-        }
-      }, true);
-
-      // Listen for hash changes from palette toggle
-      window.addEventListener('hashchange', function(e) {
-        if (window.location.hash.startsWith('#__palette')) {
-          e.preventDefault();
-          // Immediately restore scroll position
-          window.scrollTo(0, savedScrollY);
-          // Clean up hash without triggering another hashchange
-          history.replaceState(null, null, window.location.pathname);
-        }
-      }, false);
-
-      // Also handle palette input changes directly
-      document.addEventListener('DOMContentLoaded', function() {
-        const paletteInputs = document.querySelectorAll('input[name="__palette"]');
-        paletteInputs.forEach(function(input) {
-          input.addEventListener('change', function() {
-            window.scrollTo(0, savedScrollY);
-          });
-        });
-      });
-    })();
-  </script>
-
-  <!-- Hero Section -->
-  <section class="mdx-container">
-    <div class="container">
-      <div class="intro-message">
-        <div class="hero-logo">
-          <img src="logo/white.png" alt="Lance Logo">
-          <h1>Lance<sup>™</sup></h1>
-        </div>
-        <h3>The Open Lakehouse Format for Multimodal AI</h3>
-        <hr class="intro-divider" />
-        <ul class="list-inline">
-          <li>
-            <a href="quickstart" class="md-button md-button--primary">Get Started</a>
-          </li>
-          <li>
-            <a href="format" class="md-button">Read the Spec</a>
-          </li>
-          <li>
-            <a href="examples/python/llm_training" class="md-button">Train an LLM</a>
-          </li>
-          <li>
-            <a href="https://discord.gg/lance" class="md-button" target="_blank" rel="noopener">Join Discord</a>
-          </li>
-        </ul>
-      </div>
-    </div>
-  </section>
-
-  <!-- What is Lance Section -->
-  <section class="lance-intro-section">
-    <div class="container">
-      <div class="lance-intro-content">
-        <h2>What is Lance?</h2>
-        <p>
-          Lance is a modern, open source lakehouse format for multimodal AI. It contains a file format, table format, and catalog spec,
-          allowing you to build a complete open lakehouse on top of object storage to power your AI workflows.
-          Lance brings high-performance vector search, full-text search, random access, and feature
-          engineering capabilities to the lakehouse, while you can still get all the existing lakehouse benefits
-          like SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB, etc.)
-          and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore, etc.)
-        </p>
-        <p>
-          Learn more about Lance's technical details by reading our 
-          <a href="https://arxiv.org/abs/2504.15247" class="lance-paper-link" target="_blank" rel="noopener">research paper</a>
-          published at <em>VLDB 2025</em>.
-        </p>
-        <a href="quickstart" class="md-button md-button--primary">Read the Docs</a>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 1: Expressive Hybrid Search -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Expressive Hybrid Search</h2>
-          <p>
-            Lance enables powerful hybrid search combining vector similarity, full-text search,
-            and SQL analytics on the same dataset. All query types are accelerated by corresponding
-            secondary indexes as part of the Lance specification.
-          </p>
-          <p>
-            Run semantic search on embeddings, BM25 search on keywords, and apply complex SQL predicates -
-            all using a single table with a unified interface.
-          </p>
-          <a href="quickstart/vector-search" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/hybrid-search.png" alt="Hybrid Search Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 2: Lightning-fast Random Access -->
-  <section class="lance-feature-section reverse">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Lightning-fast Random Access</h2>
-          <p>
-            Lance delivers 100x faster random access compared to Parquet or Iceberg.
-            Unlike traditional formats, Lance maintains high performance even when
-            randomly accessing scattered rows across your entire dataset.
-          </p>
-          <p>
-            With a highly optimized file format plus efficient row-addressing and secondary indexes at table level,
-            you can access individual records across multiple files instantly,
-            making it perfect for real-time ML serving, random sampling, and interactive applications.
-          </p>
-          <a href="guide/read_and_write#random-access" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/random-access.png" alt="Random Access Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 3: Native Multimodal Data Support -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Native Multimodal Data Support</h2>
-          <p>
-            Store images, videos, audio, text, and embeddings alongside your traditional tabular data in a single unified format.
-            Lance's blob encoding efficiently handles large binary objects with lazy loading,
-            while optimized vector storage accelerates similarity search.
-          </p>
-          <p>
-            Perfect for AI/ML workloads where you need to store raw data, ML features, generated captions and embeddings
-            all together for multimodal retrieval and genAI workflows.
-          </p>
-          <a href="guide/blob" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/multimodal-data.png" alt="Multimodal Data Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 4: Data Evolution -->
-  <section class="lance-feature-section reverse">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Data Evolution > Schema Evolution</h2>
-          <p>
-            Schema evolution in most open table formats are metadata only and fast.
-            But when trying to backfill column values in existing rows, a full table rewrite is typically required.
-            Lance supports data evolution (efficient schema evolution with backfill), making it perfect for ML
-            feature engineering, embedding and media content management.
-          </p>
-          <p>
-            Adding a new column with data is as simple as writing new Lance files to the Lance table -
-            no need to rewrite your entire dataset.
-          </p>
-          <a href="guide/data_evolution" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/data-evolution.png" alt="Data Evolution Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 5: Rich Ecosystem Integration -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Rich Ecosystem Integrations</h2>
-          <p>
-            As an open format, Lance integrates seamlessly with the Python data ecosystem and modern data platforms.
-            Work with your favorite tools including Pandas, Polars, Ray and PyTorch for data processing and machine learning.
-          </p>
-          <p>
-            Connect with leading query engines like Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink/Fluss
-            to run SQL analytics and distributed processing on your Lance datasets.
-          </p>
-          <a href="integrations/" class="md-button">View Integrations</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/ecosystem-integrations.png" alt="Lance Ecosystem Integrations" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-
-{% endblock %}
-
-{% block content %}{% endblock %}
-{% block footer %}
-  {{ super() }}
-{% endblock %}
diff --git a/docs/pyproject.toml b/docs/pyproject.toml
index 4112230aec5..3aa6847e4f3 100644
--- a/docs/pyproject.toml
+++ b/docs/pyproject.toml
@@ -6,10 +6,12 @@ readme = "README.md"
 requires-python = ">=3.10,<3.11"
 dependencies = [
     "mkdocs>=1.5.0",
-    "mkdocs-material>=9.4.0",
+    "pymdown-extensions>=10.0",
+    "pygments>=2.16",
     "mkdocs-protobuf>=0.1.0",
     "mkdocs-linkcheck>=1.0.0",
-    "mkdocs-awesome-pages-plugin>=2.10.1"
+    "mkdocs-awesome-pages-plugin>=2.10.1",
+    "requests>=2.31.0"  # mkdocs-linkcheck imports it but doesn't declare it
 ]
 
 [tool.uv]
diff --git a/docs/src/assets/javascripts/nav-expand.js b/docs/src/assets/javascripts/nav-expand.js
deleted file mode 100644
index 17171f5a295..00000000000
--- a/docs/src/assets/javascripts/nav-expand.js
+++ /dev/null
@@ -1,16 +0,0 @@
-// Auto-expand sidebar navigation to 2 levels on page load.
-// Level 1 sections are already expanded by navigation.sections.
-// This expands level 2 (e.g. Operations, Models become visible)
-// but leaves level 3+ collapsed.
-document.addEventListener("DOMContentLoaded", function () {
-  // In mkdocs-material with navigation.sections, the top-level items
-  // are rendered as non-collapsible sections. The collapsible items
-  // start at the next level. We want to expand one more level.
-  //
-  // Selector: inside the primary nav, find toggle checkboxes that are
-  // exactly 2 nesting levels deep (the second-level sections).
-  var toggles = document.querySelectorAll(
-    ".md-sidebar--primary .md-nav--primary > .md-nav__list > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item > .md-nav__toggle"
-  );
-  toggles.forEach(function (t) { t.checked = true; });
-});
diff --git a/docs/src/assets/stylesheets/home.css b/docs/src/assets/stylesheets/home.css
deleted file mode 100644
index eafdd5afbf9..00000000000
--- a/docs/src/assets/stylesheets/home.css
+++ /dev/null
@@ -1,244 +0,0 @@
-/* Lance Homepage Styles */
-
-/* Override with custom color #625EFF site-wide */
-:root > * {
-  --md-primary-fg-color: #625EFF;
-  --md-primary-fg-color--light: #8481FF;
-  --md-primary-fg-color--dark: #4A46CC;
-  --md-accent-fg-color: #625EFF;
-  --md-accent-fg-color--transparent: rgba(98, 94, 255, 0.1);
-}
-
-* {
-  box-sizing: border-box;
-}
-
-.container {
-  width: 100%;
-  max-width: 1140px;
-  margin-right: auto;
-  margin-left: auto;
-  padding-right: 15px;
-  padding-left: 15px;
-}
-
-/* Hero Section - Fullscreen with Background Image */
-.mdx-container {
-  text-align: center;
-  color: #f8f8f8;
-  background: url("../images/lance-mj.png") no-repeat center center;
-  background-size: cover;
-  min-height: 100vh;
-  height: 100vh;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.intro-message {
-  position: relative;
-  padding: 40px 20px;
-  font-family: "Lato", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
-  max-width: 1000px;
-  margin: 0 auto;
-}
-
-.hero-logo {
-  display: inline-flex;
-  align-items: center;
-  margin-bottom: 16px;
-}
-
-.hero-logo img {
-  height: 120px;
-  width: auto;
-  margin-right: 24px;
-  margin-top: 12px;
-  filter: drop-shadow(3px 3px 8px rgba(0, 0, 0, 0.9));
-}
-
-.intro-message h1 {
-  font-weight: 400;
-  margin: 0;
-  display: inline-block;
-  text-shadow: 3px 3px 8px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1);
-  font-size: 8em;
-  line-height: 1.2;
-  color: #ffffff;
-  vertical-align: middle;
-}
-
-.intro-message h1 sup {
-  font-size: 2rem;
-  text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9);
-}
-
-.intro-message h3 {
-  font-size: 1.1rem;
-  text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1);
-  font-weight: 600;
-  margin-bottom: 32px;
-  color: #ffffff;
-}
-
-.intro-divider {
-  width: 400px;
-  max-width: 80%;
-  border-top: 1px solid rgba(255, 255, 255, 0.8);
-  border-bottom: 1px solid rgba(0, 0, 0, 0.2);
-  margin: 24px auto;
-}
-
-.list-inline {
-  padding-left: 0;
-  margin-left: -5px;
-  list-style: none;
-  margin-bottom: 0;
-}
-
-.list-inline li {
-  display: inline-block;
-  padding-right: 5px;
-  padding-left: 5px;
-}
-
-.intro-message .md-button {
-  margin: 8px;
-  padding: 14px 36px;
-  font-size: 1.1rem;
-  font-weight: 600;
-  text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.8);
-  transition: all 0.3s ease;
-}
-
-.intro-message .md-button:hover {
-  transform: translateY(-2px);
-  box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3);
-}
-
-.intro-message .md-button--primary:hover {
-  box-shadow: 0 8px 20px rgba(98, 94, 255, 0.5);
-}
-
-.intro-message .md-button:not(.md-button--primary):hover {
-  box-shadow: 0 8px 20px rgba(255, 255, 255, 0.4);
-}
-
-/* What is Lance Section */
-.lance-intro-section {
-  padding: 80px 0;
-  background-color: rgba(128, 128, 128, 0.03);
-  border-bottom: 1px solid rgba(128, 128, 128, 0.1);
-}
-
-.lance-intro-content {
-  max-width: 900px;
-  margin: 0 auto;
-  text-align: center;
-}
-
-.lance-intro-content h2 {
-  font-size: 36px;
-  font-weight: 500;
-  margin-bottom: 32px;
-  color: var(--md-primary-fg-color);
-}
-
-.lance-intro-content p {
-  font-size: 16px;
-  line-height: 1.8;
-  margin-bottom: 32px;
-  opacity: 0.9;
-  text-align: left;
-}
-
-.lance-paper-link {
-  color: var(--md-primary-fg-color);
-  text-decoration: none;
-}
-
-.lance-paper-link:hover {
-  color: var(--md-primary-fg-color);
-  text-decoration: none;
-}
-
-.lance-intro-content a:hover {
-  color: #757575;
-  text-decoration: none;
-}
-
-.lance-intro-content .md-button {
-  margin-top: 16px;
-  padding: 10px 28px;
-  font-size: 14px;
-  border: 2px solid currentColor;
-  background-color: transparent;
-  transition: all 0.3s ease;
-}
-
-.lance-intro-content .md-button:hover {
-  color: var(--md-primary-fg-color);
-  background-color: transparent;
-}
-
-/* Feature Sections */
-.lance-feature-section {
-  padding: 80px 0;
-  border-bottom: 1px solid rgba(128, 128, 128, 0.1);
-}
-
-.lance-feature-section:last-child {
-  border-bottom: none;
-}
-
-.lance-feature-content {
-  display: flex;
-  flex-wrap: wrap;
-  align-items: center;
-  gap: 60px;
-}
-
-.lance-feature-text {
-  flex: 1;
-  min-width: 300px;
-}
-
-.lance-feature-text h2 {
-  font-size: 30px;
-  font-weight: 500;
-  margin-bottom: 16px;
-  color: var(--md-primary-fg-color);
-}
-
-.lance-feature-text p {
-  font-size: 15px;
-  line-height: 1.6;
-  opacity: 0.85;
-  margin-bottom: 16px;
-}
-
-.lance-feature-text .md-button {
-  font-size: 0.6rem;
-  padding: 0;
-  transition: all 0.3s ease;
-}
-
-.lance-feature-text .md-button:hover {
-  transform: translateX(4px);
-  color: var(--md-primary-fg-color);
-}
-
-.lance-feature-demo {
-  flex: 1;
-  min-width: 400px;
-  display: flex;
-  justify-content: center;
-  overflow: hidden;
-}
-
-/* Alternating layout */
-.lance-feature-section.reverse .lance-feature-content {
-  flex-direction: row-reverse;
-}
-
-
diff --git a/docs/src/format/CLAUDE.md b/docs/src/format/CLAUDE.md
new file mode 120000
index 00000000000..47dc3e3d863
--- /dev/null
+++ b/docs/src/format/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/docs/src/format/file/encoding.md b/docs/src/format/file/encoding.md
index 4ca053d4fa6..1cb83d581e9 100644
--- a/docs/src/format/file/encoding.md
+++ b/docs/src/format/file/encoding.md
@@ -327,6 +327,148 @@ The protobuf for the full zip layout describes the compression of the data buffe
 size of the control words and how many bits we have per value (for fixed-width data) or how many bits we
 have per offset (for variable-width data).
 
+### Sparse Page Layout
+
+Sparse pages require Lance 2.3. They represent flat or nested Arrow structure directly as slot-domain mappings instead
+of dense repetition and definition events. Writers emit this layout only in files declared as 2.3 or above. The layout is
+identified only by `PageLayout`; field metadata does not identify the layout of an existing page.
+
+A domain is a layer-local integer coordinate space `[0, num_slots)`, and a slot is one element in that space. The
+outer-most domain contains the page's top-level rows. Each layer maps its parent domain to the next layer's parent
+domain, and the terminal child domain contains the leaf value slots stored in value chunks.
+
+Structural layers are ordered from outer-most to inner-most:
+
+- validity maps a nullable item or struct slot to valid or null
+- list maps non-empty parent slots to variable-size child ranges
+- fixed-size-list maps each parent slot to a child range of a fixed dimension
+
+The layer list may be empty for a flat, non-nullable leaf page. In that case the scheduling domain and
+`num_visible_items` must be equal. The explicit writer currently emits its normalized all-valid layer even when a flat
+page could use this shorter wire representation.
+
+A list slot that is valid and absent from `non_empty_positions` is an empty list. Maps use the same structural contract
+as lists. The terminal child-domain size equals `SparseLayout.num_visible_items`.
+
+`SparseLayout.num_visible_items` is the number of leaf value slots encoded in value chunks. Null leaf slots count
+because they still occupy positions in Arrow's leaf value buffer; a nullable primitive with 100 slots, including 30
+nulls, has 100 visible items. `SparseLayout.num_items` is the number of entries in the equivalent dense repetition and
+definition stream. It equals `num_visible_items` plus one structural placeholder for every list slot without children.
+The first layer's `num_slots` is the logical top-level row count used for projection.
+
+Position sets have four semantic representations: `empty`, `all`, one non-empty `range`, or an `explicit`
+delta-compressed `u64` buffer. Count sets are `empty`, one positive `constant` value, or an `explicit` compressed `u64`
+buffer. Every layer has a `SparseValiditySet` whose meaning is explicit:
+
+- `SPARSE_VALIDITY_NULL_POSITIONS`: stored positions are null and all other positions are valid
+- `SPARSE_VALIDITY_VALID_POSITIONS`: stored positions are valid and all other positions are null
+
+The unspecified validity meaning is invalid. Both polarities are part of the wire contract and have identical Arrow
+semantics after normalization.
+
+#### Writer Selection
+
+Writers may emit this layout only for Lance 2.3+ fields. A field can request it explicitly with
+`lance-encoding:structural-encoding=sparse`; the same request is an input error for earlier file versions. Without an
+explicit structural encoding, the Lance 2.3 writer selects sparse only when the dense mini-block repetition/definition
+budget would split the page or one top-level row exceeds that budget, and only when the value path is supported by the
+sparse writer. Explicit `miniblock`, `fullzip`, and `sparse` requests are not changed by this automatic policy. Lance
+2.2 and earlier writers never select sparse.
+
+Unsupported sparse value paths, including dictionary values and variable-width packed structs, retain their dense
+behavior. Writers normalize Arrow validity and list structure once. Within-budget dense pages do not build sparse
+position/count plans. All-valid layers use null positions plus `empty`; all-null layers use valid positions plus
+`empty`. Other layers choose the validity polarity with the lower semantic encoded cost, with ties using null
+positions. Field metadata controls writer selection only: readers always use `PageLayout` to determine the layout of
+an encoded page and must not use field metadata for that decision.
+
+Pages without a value payload keep the existing canonical `ConstantLayout`: structural-only types such as an empty
+struct, and leaf pages whose visible values are all null, do not emit `SparseLayout`. An explicitly sparse page with
+at least one non-null visible value does emit `SparseLayout`, even when all non-null values are equal. This boundary
+avoids introducing a second structural-only representation without evidence that it improves the existing constant
+encoding.
+
+#### Buffers and Selective Reads
+
+A sparse page contains the following physical buffers:
+
+| Buffer | Contents |
+| ------ | -------- |
+| 0 | Value chunk metadata, one 8-byte entry per chunk |
+| 1 | Mini-block compressed value chunks without repetition or definition levels |
+| 2+ | One buffer for each explicit position or count set, in structural-layer field order |
+
+Each value chunk metadata entry stores `(chunk_size / 8) - 1` as little-endian `u32`, followed by its visible value
+count as little-endian `u32`. Chunk sizes must be positive multiples of 8 and fit this representation. The sum of
+chunk sizes must equal buffer 1 exactly and the sum of chunk value counts must equal `num_visible_items`. A value chunk
+contains at most 32,768 visible values. `num_buffers` describes the number of value buffers inside every chunk and
+excludes the structural buffers.
+
+General-compressed sparse buffers use the existing length-prefixed LZ4 or Zstd representation and must not contain
+another general-compression wrapper. SparseLayout does not impose additional size or descriptor-complexity limits on
+otherwise representable buffers.
+
+Readers normalize structural metadata once, project requested top-level ranges through each layer, and read only value
+chunks that intersect the resulting leaf ranges. When no leaf range remains, readers rebuild offsets and validity from
+the structural plan without reading buffer 1.
+
+#### Caching and Point Reads
+
+Reader initialization loads buffer 0 and every explicit structural buffer, then validates and normalizes them into a
+cached page plan. The cached state contains parsed value-chunk descriptors and prefix offsets, decoded semantic
+position/count sets, validity, and the ordered structural layers. It does not contain value payload bytes from buffer
+1. The plan is cached per field and page and reused by later scans, range reads, and takes.
+
+After that plan is cached, reading one primitive leaf value reads only the value chunk that contains it. A cold read
+first loads the structural metadata and then the intersecting value chunk. Reading one top-level list or
+fixed-size-list value may intersect multiple leaf chunks and reads each intersecting chunk. A selection whose projected
+structure contains no leaf slots reads no value chunk.
+
+#### Validation
+
+Readers must reject malformed sparse metadata instead of inferring or repairing it. Required checks include:
+
+- physical buffer count, chunk-count bounds, and every checked offset/size range
+- first-layer row domain, adjacent parent/child domain chaining, and terminal visible-value domain
+- semantic set cardinality, explicit position ordering and bounds, and validity meaning
+- exact `num_items`
+- list non-empty positions being valid, count cardinality, positive counts, and child-count sum
+- fixed-size-list dimension and checked child-domain multiplication
+- value chunk byte/value sums, size representation and alignment, general-compression headers, descriptor buffer
+  count, and complete chunk consumption
+
+```protobuf
+%%% proto.message.SparseLayout %%%
+```
+
+```protobuf
+%%% proto.message.SparseStructuralLayer %%%
+```
+
+```protobuf
+%%% proto.message.SparseValidityLayer %%%
+```
+
+```protobuf
+%%% proto.message.SparseListLayer %%%
+```
+
+```protobuf
+%%% proto.message.SparseFixedSizeListLayer %%%
+```
+
+```protobuf
+%%% proto.message.SparseValiditySet %%%
+```
+
+```protobuf
+%%% proto.message.SparsePositionSet %%%
+```
+
+```protobuf
+%%% proto.message.SparseCountSet %%%
+```
+
 ### Constant Page Layout
 
 This layout is used when all (visible) values in the page are the same scalar value.
@@ -549,7 +691,7 @@ options. However, they can also be set in the field metadata in the schema.
 | `lance-encoding:dict-values-compression-level` | Integers (scheme dependent) | Varies by scheme | Compression level for dictionary values general compression                             |
 | `lance-encoding:general`             | `off`, `on`                          | `off`            | Whether to apply general compression.                                                   |
 | `lance-encoding:packed`              | Any string                           | Not set          | Whether to apply packed struct encoding (see above).                                    |
-| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`               | Not set          | Force a particular structural encoding to be applied (only useful for testing purposes) |
+| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`, `sparse`     | Not set          | Force a structural encoding; `sparse` requires Lance 2.3.                                |
 
 ### Configuration Details
 
diff --git a/docs/src/format/file/versioning.md b/docs/src/format/file/versioning.md
index 2add82ef2fa..8024a46fbc8 100644
--- a/docs/src/format/file/versioning.md
+++ b/docs/src/format/file/versioning.md
@@ -5,10 +5,10 @@ major number is changed when the file format itself is modified while the minor
 strategy is modified. Newer versions will typically have better performance and compression but may not be readable
 by older versions of Lance.
 
-In addition, the `next` alias points to an unstable format version and should not be used for production use cases.
-Breaking changes could be made to unstable encodings and that would mean that files written with these encodings are
-no longer readable by any newer versions of Lance. The `next` version should only be used for experimentation and
-benchmarking upcoming features.
+Any version explicitly labeled unstable, including the current 2.3 format and the `next` alias, should not be used for
+production use cases. Unstable formats have no compatibility guarantee: breaking encoding changes may make files
+written by one Lance build unreadable by later builds. They should only be used for experimentation and benchmarking
+upcoming features.
 
 The `stable` and `next` aliases are resolved by the specific Lance release you are using. During a format rollout
 (for example, 2.3), prefer explicit version pinning for deterministic behavior across environments.
@@ -21,7 +21,7 @@ The following values are supported:
 | 2.0            | 0.16.0                | Any                   | Rework of the Lance file format that removed row groups and introduced null support for lists, fixed size lists, and primitives |
 | 2.1            | 0.38.1                | Any                   | Enhances integer and string compression, adds support for nulls in struct fields, and improves random access performance with nested fields. |
 | 2.2            | None                  | Any                   | Adds support for newer nested type/encoding capabilities (including map support) and 2.2-era storage features. |
-| 2.3 (unstable) | None                  | Any                   | Adds experimental encodings for upcoming features. |
+| 2.3 (unstable) | None                  | Unspecified           | Adds sparse structural pages and other experimental encodings. |
 | legacy         | N/A                   | N/A                   | Alias for 0.1 |
 | stable         | N/A                   | N/A                   | Alias for the default version for new datasets in the Lance release you are running. |
 | next           | N/A                   | N/A                   | Alias for the latest unstable version in the Lance release you are running.|
diff --git a/docs/src/format/index/index.md b/docs/src/format/index/index.md
index 4a2e33c60a4..af8fdf595b4 100644
--- a/docs/src/format/index/index.md
+++ b/docs/src/format/index/index.md
@@ -177,7 +177,7 @@ or updated. These should be filtered out during query execution.
   </figcaption>
 </figure>
 
-There are three situations to consider:
+There are four situations to consider:
 
 1. **A fragment has some deleted rows.** A few of the rows in the fragment have been marked
    as deleted, but some of the rows are still present. The row addresses from the deletion
@@ -188,6 +188,17 @@ There are three situations to consider:
 3. **A fragment has had the indexed column updated in place.** This cannot be detected just
    by examining metadata. To prevent reading invalid data, the engine should filter out any
    row addresses that are not in the index's current `fragment_bitmap`.
+4. **A fragment has an updated value in an [overlay file](../table/data_overlay_file.md).**
+   This can be detected by checking if any of the fragments in the index's `fragment_bitmap`
+   have overlay files. For each overlay whose `committed_version` is greater than the index
+   segment's `dataset_version`, the overlay carries updated values not reflected in the index,
+   so its covered rows must be excluded from index results. Excluded rows are re-evaluated
+   against their current (overlaid) values on the flat path — dropping them without
+   re-evaluation would silently lose rows that match under the new value. Exclusion is
+   field-aware: only overlays covering the indexed field matter. You may exclude just the
+   affected rows or the whole fragment; the latter is simpler and safer but re-evaluates more
+   rows than necessary. See [Data Overlay Files](../table/data_overlay_file.md#index-integration)
+   for the exclusion set, re-evaluation, and correctness invariant.
 
 ## Compaction and remapping
 
diff --git a/docs/src/format/index/scalar/bloom_filter.md b/docs/src/format/index/scalar/bloom_filter.md
index 5a0e08e4228..6d924f8705d 100644
--- a/docs/src/format/index/scalar/bloom_filter.md
+++ b/docs/src/format/index/scalar/bloom_filter.md
@@ -4,6 +4,9 @@ Bloom filters are probabilistic data structures that allow for fast membership t
 They are space-efficient and can test whether an element is a member of a set.
 It's an inexact filter - they may include false positives but never false negatives.
 
+In addition, since finding NULLs is a common query pattern, the index also maintains a
+bitmap of null rows which allows it to return exact results for IS NULL queries.
+
 ## Index Details
 
 ```protobuf
@@ -32,6 +35,13 @@ The bloom filter index stores zone-based bloom filters in a single file:
 |---------------------------|--------|-------------------------------------------------------------|
 | `bloomfilter_item`        | String | Expected number of items per zone (default: "8192")         |
 | `bloomfilter_probability` | String | False positive probability (default: "0.00057", ~1 in 1754) |
+| `null_bitmap`             | UInt32 | Index of null bitmap global buffer                          |
+
+### Global Buffers
+
+| Metadata Key        | Description                                                |
+|---------------------|------------------------------------------------------------|
+| `null_bitmap`       | A serialized RowAddrTreeMap specifying which rows are null |
 
 ## Bloom Filter Spec
 
@@ -122,10 +132,11 @@ Offset 60-63:  Block 1, Word 7 (32-bit LE)
 
 ## Accelerated Queries
 
-The bloom filter index provides inexact results for the following query types:
+The bloom filter index provides inexact results for the following query types (nullability queries
+return exact results):
 
 | Query Type | Description               | Operation                                 | Result Type |
 |------------|---------------------------|-------------------------------------------|-------------|
 | **Equals** | `column = value`          | Tests if value exists in bloom filter     | AtMost      |
 | **IsIn**   | `column IN (v1, v2, ...)` | Tests if any value exists in bloom filter | AtMost      |
-| **IsNull** | `column IS NULL`          | Returns zones where has_null is true      | AtMost      |
\ No newline at end of file
+| **IsNull** | `column IS NULL`          | Returns zones where has_null is true      | Exact       |
diff --git a/docs/src/format/index/scalar/fts.md b/docs/src/format/index/scalar/fts.md
index adc7f94d65e..d5c75158011 100644
--- a/docs/src/format/index/scalar/fts.md
+++ b/docs/src/format/index/scalar/fts.md
@@ -43,6 +43,8 @@ An FTS index may contain multiple partitions. Each partition has its own set of
 | `_length`              | UInt32                  | false    | Number of documents containing the token                         |
 | `_compressed_position` | List<List<LargeBinary>> | true     | Optional compressed position lists for phrase queries            |
 
+The posting-list file schema metadata includes `posting_block_size`, the number of documents encoded per compressed posting block. Older indexes that do not have this metadata use the legacy block size `128`.
+
 ### Metadata File Schema
 
 The metadata file contains JSON-serialized configuration and partition information:
@@ -67,6 +69,7 @@ The metadata file contains JSON-serialized configuration and partition informati
 | `min_gram`          | UInt32  | 2         | Minimum n-gram length (only for ngram tokenizer)               |
 | `max_gram`          | UInt32  | 15        | Maximum n-gram length (only for ngram tokenizer)               |
 | `prefix_only`       | Boolean | false     | Generate only prefix n-grams (only for ngram tokenizer)        |
+| `block_size`        | UInt32  | 128       | Documents per compressed posting block. Must be 128 or 256. Missing values from older indexes read as 128. `256` is experimental and may introduce breaking changes. |
 
 ## Tokenizers
 
diff --git a/docs/src/format/index/scalar/zonemap.md b/docs/src/format/index/scalar/zonemap.md
index 256edc2671d..552d476cc86 100644
--- a/docs/src/format/index/scalar/zonemap.md
+++ b/docs/src/format/index/scalar/zonemap.md
@@ -8,6 +8,9 @@ zones that cannot contain matching values.
 Zone maps are "inexact" filters - they can definitively exclude zones but may include
 false positives that require rechecking.
 
+In addition, since finding NULLs is a common query pattern, the index also maintains a
+bitmap of null rows which allows it to return exact results for IS NULL queries.
+
 ## Index Details
 
 ```protobuf
@@ -34,17 +37,25 @@ The zone map index stores zone statistics in a single file:
 
 ### Schema Metadata
 
-| Key             | Type   | Description                               |
-|-----------------|--------|-------------------------------------------|
-| `rows_per_zone` | String | Number of rows per zone (default: "8192") |
+| Key                 | Type   | Description                               |
+|---------------------|--------|-------------------------------------------|
+| `rows_per_zone`     | String | Number of rows per zone (default: "8192") |
+| `null_bitmap`       | UInt32 | Index of null bitmap global buffer        |
+
+### Global Buffers
+
+| Metadata Key        | Description                                                |
+|---------------------|------------------------------------------------------------|
+| `null_bitmap`       | A serialized RowAddrTreeMap specifying which rows are null |
 
 ## Accelerated Queries
 
-The zone map index provides inexact results for the following query types:
+The zone map index provides inexact results for the following query types (nullability queries
+return exact results):
 
 | Query Type | Description               | Operation                                   | Result Type |
 |------------|---------------------------|---------------------------------------------|-------------|
 | **Equals** | `column = value`          | Includes zones where min ≤ value ≤ max      | AtMost      |
 | **Range**  | `column BETWEEN a AND b`  | Includes zones where ranges overlap         | AtMost      |
 | **IsIn**   | `column IN (v1, v2, ...)` | Includes zones that could contain any value | AtMost      |
-| **IsNull** | `column IS NULL`          | Includes zones where null_count > 0         | AtMost      |
\ No newline at end of file
+| **IsNull** | `column IS NULL`          | Includes zones where null_count > 0         | Exact       |
diff --git a/docs/src/format/index/system/mem_wal.md b/docs/src/format/index/system/mem_wal.md
index 44515f0af28..2f0a5cb1fa5 100644
--- a/docs/src/format/index/system/mem_wal.md
+++ b/docs/src/format/index/system/mem_wal.md
@@ -1,7 +1,7 @@
 # MemWAL Index
 
 The MemWAL Index is a system index that serves as the centralized structure for all MemWAL metadata.
-It stores configuration (shard specs, indexes to maintain), merge progress, and shard state snapshots.
+It stores configuration (shard specs, indexes to maintain), SSTable compaction progress, and shard state snapshots.
 
 A table has at most one MemWAL index.
 The table may be a primary-key table or an append-only table without primary-key metadata.
diff --git a/docs/src/format/table/.pages b/docs/src/format/table/.pages
index 16c20058608..5b0cb0e95e6 100644
--- a/docs/src/format/table/.pages
+++ b/docs/src/format/table/.pages
@@ -6,4 +6,5 @@ nav:
   - Layout: layout.md
   - Branch & Tag: branch_tag.md
   - Row ID & Lineage: row_id_lineage.md
+  - Data Overlay Files: data_overlay_file.md
   - MemTable & WAL: mem_wal.md
diff --git a/docs/src/format/table/data_overlay_file.md b/docs/src/format/table/data_overlay_file.md
new file mode 100644
index 00000000000..9da739a29e3
--- /dev/null
+++ b/docs/src/format/table/data_overlay_file.md
@@ -0,0 +1,388 @@
+# Data Overlay Files
+
+!!! warning "Experimental"
+
+    This feature is currently experimental and not yet supported in any library.
+
+<!-- TODO: When overlay file support is implemented, update this note to state
+     the released version that first supports the feature (and drop the
+     "experimental" framing once it is stable). -->
+
+!!! note "Overlay files require feature flag 64 (data overlay files)"
+
+    A reader or writer that does not understand overlay files must refuse a
+    dataset that uses them. Silently ignoring an overlay would return stale base
+    values, which is a correctness bug rather than a degraded experience.
+
+Overlay files supply new values for a subset of `(row offset, field)` cells
+within a fragment **without rewriting the fragment's base data files**. They make
+updates cheap when only a small fraction of rows and/or columns change: instead
+of rewriting whole columns or moving rows to a new fragment, a writer appends a
+small file carrying just the changed cells.
+
+This is Lance's third mechanism for changing data in place, alongside
+[deletion files](index.md#deletion-files) (which remove rows) and
+[data evolution](index.md#data-evolution) (which adds or rewrites whole columns).
+An overlay changes individual cells.
+
+## Concepts
+
+### Coverage and resolution
+
+Each overlay declares which cells it provides through a **coverage** bitmap (or,
+for sparse overlays, one bitmap per field). The bitmaps index **physical row
+offsets**. They include deleted rows and are stable even as deletion vectors change.
+
+To resolve a cell `(offset, field)` on read, walk the fragment's overlays from
+**newest to oldest**. The first overlay that covers `(offset, field)` wins; its
+value is used. If no overlay covers the cell, the value falls through to the base
+data file (or is `NULL` if no base data file holds that field).
+
+Precedence among overlays is determined by:
+
+1. `committed_version` — higher wins (see [Versioning](#versioning-and-ordering)).
+2. Position in `DataFragment.overlays` as a tiebreaker — a later entry is newer.
+
+A covered offset whose value is `NULL` overrides the cell **to** `NULL`. This is
+distinct from an offset that is simply absent from the bitmap, which falls
+through to the base. Coverage, not value-nullness, decides whether an overlay
+applies.
+
+### Interaction with deletions
+
+Deletions take precedence over overlays. If a row offset is marked deleted in the
+fragment's deletion file, any overlay value for that offset is dead and is
+ignored, regardless of commit order.
+
+### Physical layout
+
+An overlay's data file stores **one value column per field**, in the order of
+`data_file.fields`. It does **not** store a row-offset key column. The position of
+a covered offset's value within its column is the **rank** of that offset in the
+field's coverage bitmap — the number of set bits below it. Resolving a cell is a
+rank lookup plus one value fetch, with no separate offset column to store or
+search.
+
+Because different fields may cover different offset sets, the value columns of a
+single sparse overlay may have **different lengths**. The Lance file format
+permits columns of differing item counts within one file, so a sparse overlay is
+representable as a single file. (See [Writer support](#writer-support) for the
+current implementation status.)
+
+### Dense vs. sparse overlays
+
+A single overlay is one of two shapes:
+
+- **Dense (rectangular).** One `shared_offset_bitmap` applies to every field. Every
+  covered offset has a value for every field. This is the common case for a plain
+  `UPDATE`, where one `SET` list is applied to one set of rows.
+- **Sparse.** A `FieldCoverage` carries one bitmap per field, used when different
+  fields cover different offset sets — for example a `MERGE` with multiple
+  `WHEN MATCHED` branches, where different rows update different columns. A dense
+  overlay would have to widen to the bounding rectangle and fill the untouched
+  cells with their current values (post-images), which for wide columns such as
+  embeddings means re-storing data that did not change. A sparse overlay stores
+  exactly the changed cells.
+
+## Protobuf
+
+<details>
+<summary>DataOverlayFile protobuf message</summary>
+
+```protobuf
+%%% proto.message.DataOverlayFile %%%
+```
+
+</details>
+
+<details>
+<summary>FieldCoverage protobuf message</summary>
+
+```protobuf
+%%% proto.message.FieldCoverage %%%
+```
+
+</details>
+
+## Versioning and ordering
+
+Overlays reuse the dataset version as their ordering clock rather than
+introducing a separate generation counter.
+
+`committed_version` is the dataset version at which an overlay **became
+effective** — the version of the commit that introduced it, **not** the version
+it was read from. It is stamped at commit time and re-stamped if the commit is
+retried, in the same way as the created-at / last-updated-at version sequences.
+
+This single value drives every ordering decision:
+
+- **Overlay vs. overlay** (read precedence): higher `committed_version` wins.
+- **Overlay vs. index** (query correctness): an index records the
+  `dataset_version` it was built from. An index whose `dataset_version >=
+  committed_version` already incorporates the overlay. An overlay whose
+  `committed_version > index.dataset_version` is newer than the index and its
+  cells must be excluded from index results and re-evaluated.
+- **Scheduler signal**: the gap between an overlay's `committed_version` and an
+  index's `dataset_version`, or between an overlay and the base, is a staleness
+  measure the compaction scheduler can use.
+
+!!! note "Why effective version, not read version"
+
+    Suppose an overlay reads version 5 and commits at version 6, while an index
+    is built reading version 5 (before the overlay) and commits at version 7 with
+    `dataset_version = 5`. If the overlay stored its *read* version (5), the test
+    `5 > 5` is false, the row would not be excluded, and the index — which never
+    saw the overlay — would return a stale result. Storing the *effective*
+    version (6) makes `6 > 5` true, the cell is excluded and re-evaluated, and the
+    result is correct.
+
+## Index integration
+
+Building an index over a fragment that has overlays does **not** require dropping
+the fragment from the index's coverage. The fragment stays indexed, and the query
+path reconciles overlays at query time using an **exclusion set**.
+
+The exclusion set for an index on field `F` is the union of the coverage bitmaps,
+restricted to field `F`, of every overlay whose `committed_version >
+index.dataset_version`. The exclusion is **field-aware**: an overlay that touches
+only unrelated columns does not exclude anything from the index on `F`.
+
+The query then proceeds as:
+
+1. Run the index search as usual, producing candidate rows.
+2. Remove any candidate in the exclusion set. (Its indexed value may be stale.)
+3. **Re-evaluate** the excluded rows against their current values — the same flat
+   path already used for the unindexed tail of fragments. For a scalar predicate
+   this re-applies the filter; for a vector query it re-scores the row's current
+   vector. Rows that still match are added back to the result.
+
+Step 3 is what makes exclusion correct rather than merely safe: removing a row
+from index candidates without re-evaluating it would silently drop a row that
+should match under its new value.
+
+Exclusion is always *sufficient* because a write changes a cell only by adding an
+overlay, and that overlay's `committed_version` — the version of the commit that
+adds it — necessarily exceeds the `dataset_version` of any pre-existing index. So
+every cell a write changes is guaranteed to fall in that index's exclusion set.
+Compaction may remove an overlay only if no index still relies on it for exclusion
+(see [Compaction](#compaction)).
+
+## Compaction
+
+Overlays accumulate read cost — every overlay is a bitmap to test, a possible
+file to open, and additional work to interleave values. Compaction bounds that cost in two modes:
+
+- **Overlay → overlay.** Merge several overlays into fewer, computing the
+  post-image per `(offset, field)` by walking the merged overlays newest-first.
+  The merged overlay takes the **maximum** `committed_version` of its inputs, so
+  the exclusion semantics are preserved. The merged overlays must be **contiguous
+  in `committed_version`** — with overlays at v10, v30, and v50 you cannot merge
+  just v10 and v50, because stamping the result v50 would incorrectly promote
+  v10's values above the intervening v30 for any cell v30 also covers. Indexes can
+  still be re-used, but they may now need to exclude more rows. This is cheap to
+  write and does not touch the base.
+- **Overlay → base.** Fold overlays into a fresh base data file, computing the
+  post-image for every covered cell, then clear the overlays. The base is
+  complete, so every post-image is well defined. Overlay offsets are physical, so
+  they cannot survive a rewrite that reorders rows; folding therefore materializes
+  values rather than carrying overlays forward.
+
+!!! warning "Folding an indexed field must update its index"
+
+    An overlay→base fold removes the overlay, which removes the exclusion signal
+    that kept an index correct. Folding an overlay that covers an indexed field
+    `F` is therefore equivalent to a column rewrite of `F` and must, in the same
+    commit, either rebuild the index to a `dataset_version` at least the folded
+    overlay's `committed_version`, or remove the fragment from the index's
+    coverage so the rows fall to the flat path. Otherwise the index would serve
+    stale values with no overlay to exclude them. This is the same rule that
+    already governs rewriting a column that an index is built on.
+
+When a fragment with overlays is compacted by a row-rewriting operation
+(`RewriteRows`, which produces new fragments with new row addresses), the
+overlays are folded into the new base as part of the rewrite, and existing
+[fragment-reuse remapping](row_id_lineage.md) handles the row-address changes as
+it does today.
+
+## Row lineage
+
+An overlay write updates the `last_updated_at_version` of every covered row, so
+change-data-feed and time-travel queries observe the update. Because overlays are
+addressed by physical offset, they do **not** require stable row IDs to be
+enabled; lineage updates apply only when those features are on.
+
+## Worked example
+
+The following example illustrates how overlays function across their lifecycle, to make the rules above concrete.
+
+A table `users` with stable row IDs enabled and these fields:
+
+| field id | name      | type                    |
+|----------|-----------|-------------------------|
+| 1        | id        | `int32` (primary key)   |
+| 2        | name      | `utf8`                  |
+| 3        | age       | `int32`                 |
+| 4        | embedding | `fixed_size_list<f32,4>`|
+
+Created at version 1 as a single fragment `0` with one base data file
+`data/file0.lance` holding all four columns. `physical_rows = 4`:
+
+| offset | id | name  | age | embedding        |
+|--------|----|-------|-----|------------------|
+| 0      | 1  | Alice | 30  | …                |
+| 1      | 2  | Bob   | 25  | …                |
+| 2      | 3  | Carol | 40  | …                |
+| 3      | 4  | Dave  | 22  | …                |
+
+A BTree scalar index on `age` is built at version 1, covering fragment `0`
+(`dataset_version = 1`).
+
+### Step 1 — write an overlay
+
+```sql
+UPDATE users SET age = age + 1 WHERE id IN (2, 4);   -- Bob (offset 1), Dave (offset 3)
+```
+
+This touches one field (`age`) for two rows, so the writer emits a dense overlay —
+one shared bitmap covering both offsets — and commits it as version 2. Fragment
+`0` gains:
+
+```text
+DataOverlayFile {
+  data_file: { path: "data/overlay-<uuid>.lance", fields: [3], column_indices: [0] }
+  coverage:  shared_offset_bitmap = {1, 3}
+  committed_version: 2
+}
+```
+
+The overlay file stores a single `age` column with two values, `[26, 23]`, at
+ranks `{1,3}.rank(1) = 0` and `{1,3}.rank(3) = 1`. `last_updated_at_version` is
+set to 2 for offsets 1 and 3.
+
+### Step 2 — read
+
+`SELECT id, age FROM users` reads base ages `[30, 25, 40, 22]`. For `age`
+(field 3), the overlay covers offsets 1 and 3, so `age[1]` is replaced with the
+overlay value at rank `{1,3}.rank(1) = 0` → `26`, and `age[3]` with the value at
+rank `{1,3}.rank(3) = 1` → `23`. Result ages: `[30, 26, 40, 23]`.
+
+### Step 3 — index query
+
+```sql
+SELECT * FROM users WHERE age = 26;
+```
+
+The `age` index was built at `dataset_version = 1`; the overlay's
+`committed_version` is 2. Since `2 > 1`, the overlay's coverage for `age`, `{1, 3}`,
+is the exclusion set for this query.
+
+- The index (built at v1) holds Bob's *old* `age = 25`, so a lookup for `26`
+  returns nothing from the index.
+- The whole exclusion set is re-evaluated on the flat path, not just the rows the
+  index returned. Offset 1's current `age` (26, via the overlay) matches, so Bob
+  is returned; offset 3's current `age` (23) does not match and is dropped.
+
+The mirror case `WHERE age = 25` shows exclusion preventing a stale hit: the index
+returns offset 1 (stale `25`), but offset 1 is excluded, re-evaluated to `26`, and
+correctly dropped.
+
+### Step 4 — a second, non-rectangular write
+
+```sql
+MERGE INTO users USING staged ON users.id = staged.id
+WHEN MATCHED AND staged.kind = 'rename'  THEN UPDATE SET name = staged.name        -- Carol(2), Dave(3)
+WHEN MATCHED AND staged.kind = 'embed'   THEN UPDATE SET embedding = staged.embedding -- Bob(1)
+```
+
+`name` is updated for offsets `{2, 3}` and `embedding` for offset `{1}` — different
+fields over different rows. This is a sparse overlay, committed as version 3:
+
+```text
+DataOverlayFile {
+  data_file: { path: "data/overlay-<uuid2>.lance", fields: [2, 4], column_indices: [0, 1] }
+  coverage:  field_coverage { offset_bitmaps: [ {2,3},   {1} ] }
+  //                                     name (field 2)  ^    ^ embedding (field 4)
+  committed_version: 3
+}
+```
+
+The file's `name` column has **two** values (`["Caroline", "David"]`, at
+ranks 0 and 1 of `{2,3}`) and its `embedding` column has **one** value (at rank 0
+of `{1}`) — columns of different lengths in one file.
+
+### Step 5 — read after the second write
+
+`SELECT name, age, embedding FROM users` resolves each field independently,
+newest overlay first:
+
+- `name`: the v3 overlay covers `{2,3}` → `["Alice", "Bob", "Caroline", "David"]`.
+- `age`: the v3 overlay does not cover `age`; the v2 overlay still applies at
+  offsets 1 and 3 → `[30, 26, 40, 23]`.
+- `embedding`: the v3 overlay covers `{1}` → Bob's vector is the new one, others
+  from base.
+
+Overlays from different versions coexist and apply per field.
+
+### Step 6 — compaction (overlay → base)
+
+The scheduler folds both overlays into fragment `0` at version 4, computing
+post-images for `age`, `name`, and `embedding`, and writing a new base data file
+`data/file1.lance` with those columns. In the old file, fields 2, 3, and 4 are
+marked with a tombstone (`-2`); field 1 (`id`) remains. The fragment's `overlays` list is
+cleared. Row addresses are preserved (a column rewrite, not a row rewrite), so
+stable row IDs and the deletion vector are untouched.
+
+Because the fold removed the overlay that was excluding offsets 1 and 3 from the
+`age` index, the commit must drop fragment `0` from its coverage so `age` queries
+fall to the flat path.
+
+## Guidance
+
+!!! note "This section is a stub."
+
+    The following are implementation considerations, not part of the on-disk
+    specification.
+
+### When to overlay vs. rewrite a column vs. move rows
+
+<!-- TODO: Replace the rough heuristic below with concrete thresholds once we
+     have benchmarked the crossover points between overlays, column rewrites,
+     and row moves. -->
+
+*(To be expanded.)* The choice between appending an overlay, rewriting a full
+column (data evolution), and moving updated rows to a new fragment depends on the
+fraction of rows changed, the fraction of columns changed, column width, the
+presence of indexes on the changed columns, and the accumulated overlay read
+cost. Roughly: few rows changed favors overlays; most rows in a few columns
+favors a column rewrite; most columns changed favors moving rows to a new
+fragment.
+
+### Writer support
+
+<!-- TODO: Fill in as writer implementation progresses, including the status of
+     single-file sparse overlays (independent-length columns). -->
+
+*(To be expanded.)* Dense (rectangular) overlays write with the existing
+equal-length file writer today. Sparse overlays stored as a **single** file
+require the writer to emit columns of independent lengths, which the current v2
+writer does not yet do (it advances all columns from one global row counter).
+Until that support lands, a writer can express a sparse update as multiple dense
+overlays in one transaction.
+
+### Scheduling compaction
+
+<!-- TODO: Fill in with a concrete cost/benefit policy once compaction is
+     implemented and benchmarked. -->
+
+*(To be expanded.)* The overlay→overlay and overlay→base modes have very
+different costs; a cost/benefit scheduler decides when each is worthwhile, using
+the version gap as a staleness signal.
+
+## Related specifications
+
+- [Table format overview](index.md)
+- [Transactions: DataOverlay operation](transaction.md#dataoverlay) — write path
+  and conflict semantics
+- [Row ID & Lineage](row_id_lineage.md)
+- [Index Formats: handling overlay rows](../index/index.md#handling-deleted-and-invalidated-rows)
+- [Format Versioning](versioning.md)
diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md
index 94ea4b90dc9..f9da132cf3b 100644
--- a/docs/src/format/table/index.md
+++ b/docs/src/format/table/index.md
@@ -168,6 +168,29 @@ However, this invalidates row addresses and requires rebuilding indices, which c
 
 </details>
 
+## Data Overlay Files
+
+!!! warning "Experimental"
+
+    This feature is currently experimental and not yet supported in any library.
+
+<!-- TODO: When overlay file support is implemented, update this note to state
+     the released version that first supports the feature. -->
+
+!!! note "Overlay files require feature flag 64 (data overlay files)"
+
+Overlay files supply new values for a subset of cells within
+a fragment without rewriting the base data files. They make updates cheap when only
+a small percentage of rows and/or columns change: a writer appends a small file
+carrying just the changed cells instead of rewriting whole columns or moving rows
+to a new fragment.
+
+For the full specification — coverage and resolution rules, dense vs. sparse layout,
+versioning, index integration, compaction, and a worked example — see the
+[Data Overlay Files Specification](data_overlay_file.md).
+
+
+
 ## Related Specifications
 
 ### Storage Layout
diff --git a/docs/src/format/table/mem_wal.md b/docs/src/format/table/mem_wal.md
index 8a228721123..e73e27a7b91 100644
--- a/docs/src/format/table/mem_wal.md
+++ b/docs/src/format/table/mem_wal.md
@@ -7,684 +7,682 @@ scan, point lookup, vector search and full-text search.
 
 ![MemWAL Overview](../../images/mem_wal_overview.png)
 
-A Lance table is called a **base table** under the context of the MemWAL spec.
-It may have an [unenforced primary key](index.md#unenforced-primary-key) defined in the table schema.
-Primary keys are required for primary-key lookups and last-write-wins upsert semantics,
-but append-only MemWAL tables may omit them.
+A Lance table is called the **base table** in this document.
+The base table may have an [unenforced primary key](index.md#unenforced-primary-key) in its schema.
+Primary keys are required for primary-key lookups and last-write-wins upsert semantics.
+Append-only MemWAL tables may omit a primary key.
 
-On top of the base table, the MemWAL spec defines a set of shards.
-Writers write to shards, and data in each shard is merged into the base table asynchronously.
-An index is kept in the base table for readers to quickly discover the state of all shards at a point of time.
+MemWAL adds a set of shards on top of the base table.
+Writers append to shards.
+Each shard keeps recent data in an in-memory MemTable, persists writes to a per-shard WAL, flushes MemTables as small Lance datasets, and later compacts those SSTables into the base table.
+
+The base table manifest contains one MemWAL system index entry named `__lance_mem_wal`.
+This index stores MemWAL configuration and global progress metadata inline in `IndexMetadata.index_details`.
+Each shard's own manifest remains authoritative for shard-local mutable state.
 
 ### MemWAL Shard
 
-A **MemWAL Shard** is the main unit to horizontally scale out writes.
+A **MemWAL shard** is the unit of horizontal write scaling.
+Each shard has exactly one active writer epoch at a time.
+Writers claim a shard, append WAL entries, update the in-memory MemTable, and publish SSTable generations by updating the shard manifest.
 
-Each shard has exactly one active writer at any time.
-Writers claim a shard and then write data to that shard.
-Data in each shard is expected to be merged into the base table asynchronously.
+For primary-key tables, all rows for the same primary key must map to the same shard.
+If one primary key can appear in multiple shards, asynchronous compaction order between shards can make an older row overwrite a newer row.
+Append-only tables without a primary key do not rely on last-write-wins conflict resolution and may use any deterministic shard assignment suitable for the workload.
 
-For tables with a primary key, rows of the same primary key must be written to one and only one shard.
-If two shards contain rows with the same primary key, the following scenario can cause data corruption:
+### MemWAL Index
 
-1. Shard A receives a write with primary key `pk=1` at time T1
-2. Shard B receives a write with primary key `pk=1` at time T2 (T2 > T1)
-3. The row in shard B is merged into the base table first
-4. The row in shard A is merged into the base table second
-5. The row from Shard A (older) now overwrites the row from Shard B (newer)
+The MemWAL index is a system index entry on the base table.
+It has `name = "__lance_mem_wal"`, no indexed fields, and no index files.
+`IndexMetadata.files` is `None`.
+All MemWAL index data is stored in the `MemWalIndexDetails` protobuf message in `IndexMetadata.index_details`.
 
-This violates the expected "last write wins" semantics.
-By ensuring each primary key is assigned to exactly one shard via the sharding spec,
-merge order between shards becomes irrelevant for correctness.
-Append-only tables without a primary key do not rely on last-write-wins conflict resolution
-and may shard by any deterministic append key or partitioning column.
+The index stores:
 
-See [MemWAL Shard Architecture](#shard-architecture) for the complete shard architecture.
+- **Configuration**: `sharding_specs`, `maintained_indexes`, and `writer_config_defaults`.
+- **Compaction progress**: `compacted_sstables`, the last SSTable compacted into the base table for each shard.
+- **Index catchup progress**: `index_catchup`, the compacted SSTable generation covered by each base-table index.
+- **Shard snapshots**: optional point-in-time snapshot fields for read optimization.
 
-### MemWAL Index
+Shard snapshots are not authoritative.
+Readers that need the latest shard set list `_mem_wal/` and read each shard's latest manifest.
 
-A **MemWAL Index** is the centralized structure for all MemWAL metadata on top of a base table.
-A table has at most one MemWAL index. It stores:
+## Shard Architecture
 
-- **Configuration**: Sharding specs defining how rows map to shards, and which indexes to maintain
-- **Merge progress**: Last generation merged to base table for each shard
-- **Index catchup progress**: Which merged generation each base table index has been rebuilt to cover
-- **Shard snapshots**: Point-in-time snapshot of shard states for read optimization
+![Shard Architecture](../../images/mem_wal_shard.png)
 
-The index is the source of truth for **configuration**, **merge progress** and **index catchup progress**
-Writers and mergers read the MemWAL index to get these configurations before writing.
+Within a shard, writes first enter an in-memory **MemTable** and are durably appended to the shard **write-ahead log (WAL)**.
+The MemTable is periodically **flushed** to storage as a Lance dataset.
+SSTables are asynchronously **compacted** into the base table.
 
-Each [shard's manifest](#shard-manifest) is authoritative for its own state.
-Readers may use **shard snapshots** as a read-only optimization to see a point-in-time view of shards without opening each shard manifest.
-Readers that need the latest shard set must discover shard directories in storage and read each shard's latest manifest.
+### MemTable
 
-See [MemWAL Index Details](#memwal-index-details) for the complete structure.
+A MemTable holds rows inserted into a shard before those rows are flushed to storage.
+It serves two purposes:
 
-## Shard Architecture
+1. It buffers data and per-MemTable indexes before an SSTable is written.
+2. It lets readers access data that has not been flushed yet when strong consistency is required.
 
-![Shard Architecture](../../images/mem_wal_regional.png)
+The storage format does not prescribe the in-memory MemTable layout.
+Conceptually, a MemTable is an append log of Arrow record batches.
+Later appends have larger in-memory row positions.
+For primary-key tables, in-memory reads use the largest visible row position as the newest row for a key.
 
-Within a shard, writes are stored in an **in-memory table (MemTable)**.
-It is also written to the shard's **Write-Ahead Log (WAL)** for durability guarantee.
-The MemTable is periodically **flushed** to storage based on memory pressure and other conditions.
-**Flushed MemTables** in storage are then asynchronously **merged** into the base table.
+### SSTable Generation
 
-### MemTable
+Within each shard, SSTables have monotonically increasing generation numbers starting from 1.
+When a MemTable is flushed, the resulting SSTable is assigned the shard manifest's `current_generation`, and `current_generation` advances to the next SSTable generation.
+A MemTable does not have a generation.
 
-A MemTable holds rows inserted into the shard before flushing to storage.
-It serves 2 purposes:
+SSTable generation numbers order persisted data freshness within one shard:
 
-1. build up data and related indexes to be flushed to storage as a flushed MemTable
-2. allow a reader to potentially access data that is not flushed to storage yet
+- Base table data is modeled as generation 0.
+- Higher SSTable generations are newer.
+- The active MemTable is newer than every published SSTable.
+- Within the active MemTable, higher row positions are newer.
+- Within an SSTable, flush-time deletion vectors hide older duplicate primary-key rows, so readers see at most the newest row for each primary key.
 
-#### MemTable Format
+## WAL
 
-The complete in-memory format of a MemTable is implementation-specific and out of the scope of this spec.
-The Lance core Rust SDK maintains one default implementation and is available through all its language binding SDKs,
-but integrations are free to build their own MemTable format depending on the specific use cases,
-as long as it follows the MemWAL storage layout, reader and writer requirements when flushing MemTable.
+The WAL is the durable append log for a shard.
+Every durable WAL append creates one **WAL entry**.
 
-Conceptually, because Lance uses [Arrow as its in-memory data exchange format](https://arrow.apache.org/docs/format/index.html),
-for the ease of explanation in this spec, we will treat MemTable as a list of Arrow record batches,
-and each write into the MemTable is a new Arrow record batch.
+### WAL Entry Positions
 
-#### MemTable Generation
+WAL entry positions are 1-based.
+The first data entry is position 1.
+Position 0 is reserved as the sentinel value meaning no WAL entry has been covered.
 
-Based on conditions like memory limit and durability requirements,
-a MemTable needs to be **flushed** to storage and discarded.
-When that happens, new writes go to a new MemTable and the cycle repeats.
-Each MemTable is assigned a monotonically increasing generation number starting from 1.
-When MemTable of generation `N` is discarded, the next MemTable gets assigned generation `N+1`.
+Writers append WAL entries in increasing position order.
+If entry `N` is not fully written, entry `N + 1` must not exist.
+Recovery replays from `replay_after_wal_entry_position + 1`.
 
-### WAL
+### WAL Entry Format
 
-WAL serves as the durable storage of all MemTables in a shard.
-It consists of data in MemTables ordered by generation.
-Every time we write to the WAL, we call it a **WAL Flush**.
+Each WAL entry is an Apache Arrow IPC stream file.
+The Arrow schema metadata includes:
 
-#### WAL Durability
+- `writer_epoch`: decimal string containing the writer epoch that created the entry.
+- `fence_sentinel`: optional marker for a data-less fence sentinel entry.
 
-When a write is flushed to WAL, the specific write becomes durable.
-Otherwise, if the MemTable is lost, data is also lost.
+A normal WAL entry contains one or more record batches.
+A fence sentinel entry contains no batches and is skipped during replay.
+Sentinels are used so an older writer collides on the next WAL position and discovers that it has been fenced.
 
-Multiple writes can be batched together in a single WAL flush to reduce WAL flush frequency and improve throughput.
-The more writes a single WAL flush batches, the longer it takes for a write to be durable.
+### WAL Storage Layout
 
-The whole LSM tree's durability is determined by the durability of the WAL.
-For example, if WAL is stored in Amazon S3, it has 99.999999999% durability.
-If it is stored in local disk, the data will be lost if the local disk is damaged.
+WAL entries live under `_mem_wal/{shard_id}/wal/`.
+Filenames use bit-reversed 64-bit binary names with the `.arrow` suffix:
 
-#### WAL Entry
+```text
+_mem_wal/{shard_id}/wal/{bit_reversed_position}.arrow
+```
 
-Each time a WAL flush happens, it adds a new **WAL Entry** to the WAL.
-In other words, a WAL consists of an ordered list of WAL entries starting from position 0.
-Writer must flush WAL entries in sequential order from lower to higher position.
-If WAL entry `N` is not flushed fully, WAL entry `N+1` must not exist in storage.
+The bit-reversal spreads sequential positions across object-store keyspace.
+For example, position 5 is encoded as:
 
-#### WAL Replay
+```text
+1010000000000000000000000000000000000000000000000000000000000000.arrow
+```
 
-**Replaying** a WAL means to read data in the WAL from a lower to a higher position.
-This is commonly used to recover the latest MemTable after it is lost,
-by reading from the start position of the latest MemTable generation till the highest position in the WAL,
-assuming proper fencing to guard against multiple writers to the same shard.
+## SSTable
 
-See [Writer Fencing](#writer-fencing) for the full fencing mechanism.
+An SSTable is the immutable result of flushing a MemTable.
+It is stored as a Lance dataset under its shard directory.
 
-#### WAL Entry Format
+!!! note
+    Unlike a classic LSM sorted string table, a MemWAL SSTable is not sorted by key; random access is instead served by its BTree primary-key sidecar. It is called an SSTable because it is an immutable, persisted, indexed run.
 
-Each WAL entry is a file in storage following the [Apache Arrow IPC stream format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) to store the batch of writes in the MemTable.
-The writer epoch is stored in the stream's Arrow schema metadata with key `writer_epoch` for fencing validation during replay.
+### SSTable Storage Layout
 
-#### WAL Storage Layout
+An SSTable with generation `i` is written to:
 
-Each WAL entry is stored within the WAL directory of the shard located at `_mem_wal/{shard_id}/wal`.
+```text
+_mem_wal/{shard_id}/{random8}_gen_{i}/
+```
 
-WAL files use bit-reversed 64-bit binary naming to distribute files evenly across the directory keyspace.
-This optimizes S3 throughput by spreading sequential writes across S3's internal partitions, minimizing throttling.
-The filename is the bit-reversed binary representation of the entry ID with suffix `.arrow`.
-For example, entry ID 5 (binary `000...101`) becomes `1010000000000000000000000000000000000000000000000000000000000000.arrow`.
+`{random8}` is an 8-character random hex value generated for each flush attempt.
+If a flush attempt fails, a retry writes a different directory instead of reusing a partially written one.
+The shard manifest records the successful directory name in the SSTable's `path`.
+
+The SSTable directory is a standard Lance dataset written with the base table's data storage version.
+Each SSTable is written as one fragment.
+Additional MemWAL sidecars may be present:
+
+```text
+{random8}_gen_{i}/
+├── _versions/
+│   └── {version}.manifest
+├── _deletions/                         # Present when within-SSTable dedup deletes rows
+├── _indices/                           # Present when maintained user indexes are built
+│   └── {index_uuid}/
+├── _pk_index/                          # Primary-key sidecar BTree, not a manifest index
+└── bloom_filter.bin                    # Primary-key bloom filter
+```
 
-### Flushed MemTable
+The exact Lance dataset internals follow the [Lance table storage layout](layout.md).
 
-A flushed MemTable is created by flushing the MemTable to storage.
-In Lance MemWAL spec, a flushed MemTable must be a Lance table following the Lance table format spec.
+### SSTable Row Order
 
-!!!note
-This is called Sorted String Table (SSTable) or Sorted Run in many LSM-tree literatures and implementations.
-However, since our MemTable is not sorted, we just use the term flushed MemTable to avoid confusion.
+SSTable rows are written in forward insert order.
+Physical row offsets increase with write time.
+For a duplicate primary key within one SSTable, the newest row has the largest physical offset.
 
-#### Flushed MemTable Storage Layout
+Primary-key SSTables use a deletion vector to expose last-write-wins semantics.
+During flush, the writer scans rows in forward order, keeps the last occurrence of each primary key, and marks all earlier duplicate offsets deleted.
+The deletion vector is attached to fragment 0 in the SSTable's Lance manifest.
 
-The MemTable of generation `i` is flushed to `_mem_wal/{shard_id}/{random_hex}_gen_{i}/` directory,
-where `{random_hex}` is a random 8-character hex value generated at flush time.
-The random hex value is necessary to ensure if one MemTable flush attempt fails,
-The retry can use another directory.
-The content within the generation directory follows the [Lance table storage layout](layout.md).
+Append-only SSTables without a primary key do not perform primary-key deduplication and retain every row.
 
-#### Merging MemTable to Base Table
+### Tombstone Rows
 
-Generation numbers determine merge order of flushed MemTable into base table:
-lower numbers represent older data and must be merged to the base table first to preserve correct upsert semantics.
+Delete operations are represented as rows with the internal `_tombstone` column.
+Tombstone rows follow the same forward row ordering and deletion-vector rules as ordinary rows.
+If the newest row for a primary key is a tombstone, the deletion vector keeps that tombstone row and hides older rows for the key.
+Read planning then filters `_tombstone = false`, so the key is absent from query results.
 
-Within a single flushed MemTable for a primary-key table,
-if there are multiple rows of the same primary key, the row that is last inserted wins.
-Append-only tables without a primary key retain all inserted rows.
+### SSTable Primary-Key Sidecars
 
-### Shard Manifest
+Primary-key MemTables maintain an implicit BTree for primary-key deduplication, independent of `maintained_indexes`.
+When a primary-key MemTable is flushed, the SSTable writes two primary-key sidecars:
 
-Each shard has a manifest file. This is the source of truth for the state of a shard.
+- `bloom_filter.bin` stores the SSTable's primary-key bloom filter and lets point lookups skip SSTables that cannot contain the queried key.
+- `_pk_index/` stores a standalone BTree over primary-key values to forward row ids.
 
-#### Shard Manifest Contents
+The `_pk_index/` sidecar is not a maintained user index, is not registered in the SSTable's Lance manifest, and has no manifest UUID.
+Its identity is its immutable SSTable path.
+Readers open it directly from `{sstable_path}/_pk_index`.
 
-The manifest contains:
+The `_pk_index/` directory is a Lance scalar BTree index store:
 
-- **Fencing state**: `writer_epoch` as the latest writer fencing token, see [Writer Fencing](#writer-fencing) for more details.
-- **Shard assignment**: `shard_spec_id` and `shard_field_values` record how this shard maps to its sharding spec. `shard_field_values` is a map from shard field id to the raw Arrow scalar bytes of the computed value; the matching `ShardingField.result_type` in the `ShardingSpec` determines how to interpret each entry (e.g., 4 little-endian bytes for int32, raw UTF-8 bytes for utf8).
-- **WAL pointers**: `replay_after_wal_entry_position` (last entry position flushed to MemTable, 0-based), `wal_entry_position_last_seen` (last entry position seen at manifest update, 0-based)
-- **Generation trackers**: `current_generation` (next generation to flush), `flushed_generations` list of generation number and directory path pairs (e.g., generation 1 at `a1b2c3d4_gen_1`)
+```text
+_pk_index/
+├── page_data.lance
+└── page_lookup.lance
+```
 
-Note: `wal_entry_position_last_seen` is a hint that may be stale since it's not updated on WAL write.
-It is updated opportunistically by any reader that can update the shard manifest.
-The manifest itself is atomically written, but recovery must try to get newer WAL files to find the actual state beyond this hint.
+Readers load this directory as a BTree index using `BTreeIndexDetails` with default parameters.
+The primary-key index type is the Arrow type of the primary-key column for a single-column primary key, or `Binary` for a composite primary key.
 
-The manifest is serialized as a protobuf binary file using the `ShardManifest` message.
+The `page_lookup.lance` file has the following schema:
 
-<details>
-<summary>ShardManifest protobuf message</summary>
+| Column       | Type                    | Nullable | Description                                    |
+|--------------|-------------------------|----------|------------------------------------------------|
+| `min`        | {PrimaryKeyIndexType}   | true     | Minimum primary-key index value in the page    |
+| `max`        | {PrimaryKeyIndexType}   | true     | Maximum primary-key index value in the page    |
+| `null_count` | UInt32                  | false    | Number of null values in the page              |
+| `page_idx`   | UInt32                  | false    | Page number pointing into `page_data.lance`    |
 
-```protobuf
-%%% mem_wal.message.ShardManifest %%%
-```
+The `page_data.lance` file has the following schema:
 
-</details>
+| Column   | Type                  | Nullable | Description                                                       |
+|----------|-----------------------|----------|-------------------------------------------------------------------|
+| `values` | {PrimaryKeyIndexType} | true     | Sorted primary-key index values                                   |
+| `ids`    | UInt64                | false    | Forward row ids corresponding to each primary-key index value     |
 
-#### Shard Manifest Versioning
+For a single-column primary key, the indexed value stores the primary-key scalar directly.
+For a composite primary key, the indexed value stores an order-preserving binary tuple encoding of all primary-key columns in primary-key column order.
+Each tuple column is encoded as:
 
-Manifests are versioned starting from 1 and immutable.
-Each update creates a new manifest file at the next version number.
-Updates use put-if-not-exists or file rename to ensure atomicity depending on the storage system.
-If two processes compete, one wins and the other retries.
+- `0x00` for null.
+- `0x01` followed by the non-null value encoding otherwise.
 
-To commit a manifest version:
+Supported non-null value encodings are:
 
-1. Compute the next version number
-2. Write the manifest to `{bit_reversed_version}.binpb` using put-if-not-exists
-3. In parallel best-effort write to `version_hint.json` with `{"version": <new_version>}` (failure is acceptable)
+- Signed integers and date values: sign-flipped 8-byte big-endian integer bytes.
+- Unsigned integers: 8-byte big-endian unsigned integer bytes.
+- Boolean: one byte, `0x00` for false and `0x01` for true.
+- UTF-8 and binary values: raw bytes, with each `0x00` byte escaped as `0x00 0xff`, followed by a `0x00 0x00` terminator.
 
-To read the latest manifest version:
+This encoding is injective and preserves primary-key tuple ordering under lexicographic byte comparison.
+Composite primary-key columns must use one of the supported encodings above.
 
-1. Read `version_hint.json` to get the latest version hint. If not found, start from version 1
-2. Check existence for subsequent versions from the starting version
-3. Continue until a version is not found
-4. The latest version is the last found version
+The sidecar row ids are in the same forward row-position space as the data files, deletion vector, and maintained user indexes.
+The sidecar is used for cross-generation membership and block-list checks.
+It is not used to choose the newest row inside the same SSTable; the deletion vector has already hidden older same-generation duplicates.
 
-!!!note
-This works because the write rate to shard manifests is significantly lower than read rates. Shard manifests are only updated when shard metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version.
+### Maintained User Indexes
 
-#### Shard Manifest Storage Layout
+When the MemWAL index lists `maintained_indexes`, flush may build matching indexes inside the SSTable.
+These index files live in the SSTable's `_indices/{index_uuid}/` directory and are recorded in the SSTable's Lance manifest.
+The implicit primary-key BTree sidecar is not included in `maintained_indexes` and does not live under `_indices/`.
 
-All shard manifest versions are stored in `_mem_wal/{shard_id}/manifest` directory.
+These indexes use the same row-position space as the forward-written data files.
+If the SSTable has a primary key, its deletion vector masks stale duplicate rows for indexed reads as well.
 
-Each shard manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files.
-For example, version 5 becomes `1010000000000000000000000000000000000000000000000000000000000000.binpb`.
+### SSTable Compaction
 
-## MemWAL Index Details
+SSTables are compacted into the base table in ascending generation order within each shard.
+Lower generation numbers are older and must be compacted before higher generation numbers.
+SSTable compaction uses merge-insert semantics so newer rows overwrite older rows for the same primary key.
 
-The MemWAL Index uses the [standard index storage](../index/index.md#index-storage) at `_indices/{UUID}/`.
+## Shard Manifest
 
-The index stores its data in two parts:
+Each shard has a versioned manifest.
+The latest shard manifest is the source of truth for shard-local state.
 
-1. **Index details** (`index_details` in `IndexMetadata`): Contains configuration, merge progress, and snapshot metadata
-2. **Shard snapshots**: Stored as a Lance file or inline, depending on shard count
+### Shard Manifest Contents
 
-### Index Details
+The manifest contains:
+
+- **Identity**: `shard_id`, `shard_spec_id`, and `shard_field_entries`.
+- **Fencing state**: `writer_epoch`.
+- **WAL pointers**: `replay_after_wal_entry_position` and `wal_entry_position_last_seen`.
+- **SSTable generation state**: `current_generation` and `sstables`.
+- **Lifecycle state**: `status`, either `ACTIVE` or `SEALED`.
+
+`shard_field_entries` stores computed shard field values as raw Arrow scalar bytes keyed by `ShardingField.field_id`.
+The matching `ShardingField.result_type` determines how to decode each value.
+For example, `int32` values are four little-endian bytes and `utf8` values are raw UTF-8 bytes.
+
+`replay_after_wal_entry_position` is the most recent 1-based WAL position covered by an SSTable.
+The default value 0 means no WAL entry has been covered and recovery starts at position 1.
+
+`wal_entry_position_last_seen` is a best-effort hint for the most recent WAL position observed at manifest update time.
+It is not authoritative because it is not updated on every WAL write.
+Recovery must still probe or list WAL files to find the actual tail.
 
-The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` protobuf message with the following key fields:
+`current_generation` is the generation number to assign to the next SSTable created by flushing the MemTable.
+Each entry in `sstables` records a published SSTable's `generation` and `path`.
 
-- **Configuration fields** (`sharding_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration.
-  Writers read these fields to determine how to partition data and which indexes to maintain.
-- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each shard.
-  This field is updated atomically with merge-insert data commits, enabling conflict resolution when multiple mergers operate concurrently.
-  Each entry contains the shard UUID and generation number.
-- **Index catchup progress** (`index_catchup`) tracks which merged generation each base table index has been rebuilt to cover.
-  When data is merged from a flushed MemTable to the base table, the base table's indexes may be rebuilt asynchronously.
-  During this window, queries should use the flushed MemTable's pre-built indexes instead of scanning unindexed data in the base table.
-  See [Indexed Read Plan](#indexed-read-plan) for details.
-- **Shard snapshot fields** (`snapshot_ts_millis`, `num_shards`, `inline_snapshots`) provide a snapshot of shard states.
-  The actual shard manifests remain authoritative for shard state.
-  When `num_shards` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema.
+`status = SEALED` marks a reversible in-flight drop-table operation.
+Sealed shards refuse new writer claims.
+
+The manifest is serialized as the `ShardManifest` protobuf message.
 
 <details>
-<summary>MemWalIndexDetails protobuf message</summary>
+<summary>ShardManifest protobuf message</summary>
 
 ```protobuf
-%%% mem_wal.message.MemWalIndexDetails %%%
+%%% mem_wal.message.ShardManifest %%%
 ```
 
 </details>
 
-### Shard Identifier
+### Shard Manifest Versioning
 
-Each shard has a unique UUID identifier within the table.
-When a new shard is created, implementations may assign either a random UUID or
-a deterministic UUID derived from the shard assignment when deterministic
-writer fencing is required.
+Manifest versions start at 1.
+Each update writes a new immutable protobuf file:
 
-### Shard Discovery
-
-The MemWAL index can store shard snapshots for read optimization, but those snapshots may lag the latest shard set.
-Implementations that need to discover the current shard set should list `_mem_wal/` shard directories and read each shard's latest [shard manifest](#shard-manifest).
-
-Each shard manifest records the shard UUID, sharding spec ID, and computed shard field values needed to map the shard back to a sharding spec assignment.
-
-### Sharding Spec
+```text
+_mem_wal/{shard_id}/manifest/{bit_reversed_version}.binpb
+```
 
-A **Sharding Spec** defines how all rows in a table are logically divided into different shards,
-enabling automatic shard assignment and query-time shard pruning.
+Writers use put-if-not-exists or atomic rename, depending on storage support.
+If two processes race to write the same next version, one wins and the other reloads and retries.
 
-Each sharding spec has:
+After a successful version write, the writer best-effort updates:
 
-- **Spec ID**: A positive integer that uniquely identifies this spec within the MemWAL index. IDs are never reused.
-- **Sharding fields**: An array of field definitions that determine how to compute shard values.
+```json
+{"version": <new_version>}
+```
 
-Each shard is bound to a specific sharding spec ID, recorded in its [manifest](#shard-manifest).
-Shards without a spec ID (`spec_id = 0`) are manually-created shards not governed by any spec.
+in:
 
-A sharding spec's field array consists of **sharding field** definitions.
-Each sharding field has the following properties:
+```text
+_mem_wal/{shard_id}/manifest/version_hint.json
+```
 
-| Property      | Description                                                               |
-| ------------- | ------------------------------------------------------------------------- |
-| `field_id`    | Unique string identifier for this sharding field                         |
-| `source_ids`  | Array of field IDs referencing source columns in the schema               |
-| `transform`   | A well-known shard expression, specify this or `expression`              |
-| `expression`  | A DataFusion SQL expression for custom logic, specify this or `transform` |
-| `result_type` | The output type of the shard value                                       |
+Readers use `version_hint.json` as a starting point and then probe subsequent versions until a version is missing.
+The latest manifest is the last existing version.
 
-#### Shard Expression
+## MemWAL Index Details
 
-A **Shard Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a shard value from source column(s).
-Source columns are referenced as `col0`, `col1`, etc., corresponding to the order of field IDs in `source_ids`.
+The MemWAL index is stored inline in the base table's `IndexMetadata`.
+It is a system index with no file directory.
+The `index_details` field contains a `MemWalIndexDetails` protobuf message.
 
-Shard expressions must satisfy the following requirements:
+Important fields:
 
-1. **Deterministic**: The same input value must always produce the same output value.
-2. **Stateless**: The expression must not depend on external state (e.g., current time, random values, session variables).
-3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same shard value).
-4. **Column removal resistant**: If a source field ID is not found in the schema, the column should be interpreted as NULL.
-5. **NULL-safe**: The expression should properly handle NULL inputs and have defined behavior (e.g., return NULL if input is NULL for single-column expressions).
-6. **Consistent with result type**: The expression's return type must be consistent with `result_type` in non-NULL cases.
+- `sharding_specs`: sharding configuration used by writers and shard pruning.
+- `maintained_indexes`: names of base-table indexes to maintain in MemTables and SSTables.
+- `writer_config_defaults`: string map of default writer configuration values persisted for all writers.
+- `compacted_sstables`: per-shard compaction progress, updated atomically with base-table compaction commits.
+- `index_catchup`: per-index coverage progress after data has been compacted into the base table.
+- `snapshot_ts_millis`, `num_shards`, and `inline_snapshots`: optional shard snapshot fields for read optimization.
 
-#### Shard Transform
+If a shard is absent from `index_catchup` for an index, that index is assumed to be fully caught up for the shard.
 
-A **Shard Transform** is a well-known shard expression with a predefined name.
-When a transform is specified, the expression is derived automatically.
+Shard snapshots, when present, use the following Lance file schema:
 
-| Transform      | Parameters    | Shard Expression                                         | Result Type    |
-| -------------- | ------------- | --------------------------------------------------------- | -------------- |
-| `identity`     | (none)        | `col0`                                                    | same as source |
-| `year`         | (none)        | `date_part('year', col0)`                                 | `int32`        |
-| `month`        | (none)        | `date_part('month', col0)`                                | `int32`        |
-| `day`          | (none)        | `date_part('day', col0)`                                  | `int32`        |
-| `hour`         | (none)        | `date_part('hour', col0)`                                 | `int32`        |
-| `bucket`       | `num_buckets` | `abs(murmur3(col0)) % N`                                  | `int32`        |
-| `multi_bucket` | `num_buckets` | `abs(murmur3_multi(col0, col1, ...)) % N`                 | `int32`        |
-| `truncate`     | `width`       | `left(col0, W)` (string) or `col0 - (col0 % W)` (numeric) | same as source |
+| Column                     | Type                         | Nullable | Description                                            |
+|----------------------------|------------------------------|----------|--------------------------------------------------------|
+| `shard_id`                 | Utf8                         | false    | Shard UUID string                                      |
+| `shard_spec_id`            | UInt32                       | false    | Sharding spec that produced the shard                  |
+| `shard_field_{field_id}`   | `ShardingField.result_type`  | false    | Computed shard field value for the given sharding field |
 
-The `bucket` and `multi_bucket` transforms use Murmur3 hash functions:
+The MemWAL index data is stored inline.
+Readers discover the latest shard set by listing `_mem_wal/` shard directories and reading shard manifests.
 
-- **`murmur3(col)`**: Computes the 32-bit Murmur3 hash (x86 variant, seed 0) of a single column. Returns a signed 32-bit integer. Returns NULL if input is NULL.
-- **`murmur3_multi(col0, col1, ...)`**: Computes the Murmur3 hash across multiple columns. Returns a signed 32-bit integer. NULL fields are ignored during hashing; returns NULL only if all inputs are NULL.
+<details>
+<summary>MemWalIndexDetails protobuf message</summary>
 
-The hash result is wrapped with `abs()` and modulo `N` to produce a non-negative bucket number in the range `[0, N)`.
+```protobuf
+%%% mem_wal.message.MemWalIndexDetails %%%
+```
 
-### Shard Snapshot Storage
+</details>
 
-Shard snapshots are stored using one of two strategies based on the number of shards:
+## Sharding
 
-| Shard Count       | Storage Strategy    | Location                                  |
-| ------------------ | ------------------- | ----------------------------------------- |
-| <= 100 (threshold) | Inline              | `inline_snapshots` field in index details |
-| > 100              | External Lance file | `_indices/{UUID}/index.lance`             |
+A **ShardingSpec** defines how rows map to shards.
+Each spec has a positive `spec_id` and one or more `ShardingField` entries.
+Each shard manifest records the `shard_spec_id` and the computed shard field values for that shard.
+`spec_id = 0` means the shard was manually created and is not governed by a sharding spec.
 
-The threshold (100 shards) is implementation-defined and may vary.
+Each `ShardingField` contains:
 
-**Inline storage**: For small shard counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field.
-This keeps the index metadata compact while avoiding an additional file read for common cases.
+- `field_id`: stable identifier for the computed shard field.
+- `source_ids`: field IDs of source columns in the Lance schema.
+- `transform`: well-known transform name, when using built-in transform evaluation.
+- `expression`: reserved custom expression text, mutually exclusive with `transform`.
+- `result_type`: Arrow type name for the computed value.
+- `parameters`: transform-specific string parameters.
 
-**External Lance file**: For large shard counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`.
-This file uses standard Lance format with the shard snapshot schema, enabling efficient columnar access and compression.
+The supported built-in transforms are:
 
-### Shard Snapshot Arrow Schema
+- `unsharded`: takes no source columns, always returns `int32` value 0, and creates one shard.
+- `bucket`: takes one source column and `num_buckets`, hashes the value, and returns an `int32` bucket id in `[0, num_buckets)`.
+- `identity`: takes one source column and returns the raw scalar value as the shard value.
 
-Shard snapshots are stored as a Lance file with one row per shard.
-The snapshot schema is optimized for shard discovery. Full mutable shard state
-remains in the authoritative shard manifest files.
+`bucket` computes a deterministic 32-bit hash with seed 0 and then computes:
 
-| Column                   | Type          | Description                                                                                                |
-| ------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------- |
-| `shard_id`               | `utf8`        | Shard UUID string                                                                                          |
-| `shard_spec_id`          | `uint32`      | Sharding spec ID (0 if manual)                                                                             |
-| `shard_field_{field_id}` | varies        | One column per sharding field defined in the sharding spec, typed to match the field's `ShardingField.result_type`. |
+```text
+(hash & i32::MAX) % num_buckets
+```
 
-For example, with a sharding spec containing a field `user_bucket` of type `int32`:
+`num_buckets` must be in `[1, 1024]`.
+Null bucket values hash to 0 and therefore map to bucket 0.
+See [Appendix 3: Bucket Hashing](#appendix-3-bucket-hashing) for the exact hash algorithm and test vectors.
 
-| Column                     | Type    | Description                  |
-| -------------------------- | ------- | ---------------------------- |
-| ...                        | ...     | (base columns above)         |
-| `shard_field_user_bucket`  | `int32` | Bucket value for this shard |
+The `bucket` transform supports scalar boolean, integer, floating-point, date32, time, timestamp, utf8, and large_utf8 source types.
+The `identity` transform supports scalar boolean, integer, utf8, and large_utf8 source types.
 
-This schema records the fields needed to map each shard back to its sharding spec
-assignment. Readers that need fencing epochs, WAL positions, or flushed
-generation state must read the latest shard manifests directly.
+The `year`, `month`, `day`, `hour`, `multi_bucket`, and `truncate` transform names are not supported MemWAL sharding transforms and must not be used in `ShardingSpec.transform`.
 
 ## Storage Layout
 
-Here is a recap of the storage layout with all the files and concepts defined so far:
+The MemWAL storage layout is:
 
-```
+```text
 {table_path}/
+├── _versions/
+│   └── ...                              # Base table manifests, including __lance_mem_wal index metadata
 ├── _indices/
-│   └── {index_uuid}/                    # MemWAL Index (uses standard index storage)
-│       └── index.lance                  # Serialized shard snapshots (Lance file)
-│
+│   └── ...                              # Ordinary base table index files; MemWAL index has no files
 └── _mem_wal/
-    └── {shard_id}/                   # Shard directory (UUID v4)
+    └── {shard_id}/
         ├── manifest/
-        │   ├── {bit_reversed_version}.binpb     # Serialized shard manifest (bit-reversed naming)
-        │   └── version_hint.json                # Version hint file
+        │   ├── {bit_reversed_version}.binpb
+        │   └── version_hint.json
         ├── wal/
-        │   ├── {bit_reversed_entry_id}.arrow    # WAL data files (bit-reversed naming)
+        │   ├── {bit_reversed_position}.arrow
         │   └── ...
-        └── {random_hash}_gen_{i}/        # Flushed MemTable (generation i, random prefix)
+        └── {random8}_gen_{generation}/
             ├── _versions/
-            │   └── {version}.manifest    # Table manifest (V2 naming scheme)
-            ├── _indices/                 # Indexes
-            │   ├── {vector_index}/
-            │   └── {scalar_index}/
-            └── bloom_filter.bin          # Primary key bloom filter
+            │   └── {version}.manifest
+            ├── _deletions/
+            ├── _indices/
+            │   └── {index_uuid}/
+            ├── _pk_index/
+            └── bloom_filter.bin
 ```
 
-## Implementation Expectation
-
-This specification describes the storage layout for the LSM tree architecture. Implementations are free to use any approach to fulfill the storage layout requirements. Once data is written to the expected storage layout, the reader and writer expectations apply.
+Some SSTable subdirectories are conditional.
+For example, `_deletions/` is present only when the SSTable's Lance manifest references a deletion vector, `_indices/` is present only when maintained user indexes are built, and `_pk_index/` plus `bloom_filter.bin` are meaningful for primary-key tables.
 
-The specification defines:
-
-- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, shard manifests, and the MemWAL index
-- **Durability guarantees**: How data is persisted through WAL entries and flushed MemTables
-- **Consistency model**: How readers and writers coordinate through manifests and epoch-based fencing
+## Implementation Expectation
 
-Implementations may choose different approaches for:
+This document specifies the storage layout and observable reader and writer invariants.
+Implementations may choose different in-memory structures, buffering policies, background scheduling, and query execution plans.
 
-- In-memory data structures and indexing
-- Buffering strategies before WAL flush
-- Background task scheduling and concurrency
-- Query execution strategies
+An implementation is compatible when it:
 
-As long as the storage layout is correct and the documented invariants are maintained, implementations can optimize for their specific use cases.
+1. Writes WAL entries, shard manifests, SSTables, and MemWAL index metadata using the documented layout.
+2. Preserves WAL position, writer fencing, and manifest versioning invariants.
+3. Exposes last-write-wins semantics for primary-key tables.
+4. Preserves append-only semantics for tables without primary keys.
+5. Maintains generation ordering when compacting SSTables into the base table.
 
 ## Writer Expectations
 
-A writer operates on a single shard and is responsible for:
+A writer operates on one shard and is responsible for:
 
-1. Claiming the shard using epoch-based fencing
-2. Writing data to WAL entries and flushed MemTables following the [storage layout](#storage-layout)
-3. Maintaining the shard manifest to track WAL and generation progress
+1. Claiming the shard with epoch-based fencing.
+2. Appending WAL entries in sequential 1-based positions.
+3. Maintaining in-memory MemTable state.
+4. Flushing MemTables to SSTable Lance datasets.
+5. Updating the shard manifest after an SSTable is durably written.
 
 ### Writer Fencing
 
-Writers use epoch-based fencing to ensure single-writer semantics per shard.
+Writers use `writer_epoch` to enforce single-writer semantics per shard.
 
 To claim a shard:
 
-1. Load the latest shard manifest
-2. Increment `writer_epoch` by one
-3. Atomically write a new manifest version
-4. If the write fails (another writer claimed the epoch), reload and retry with a higher epoch
+1. Load the latest shard manifest.
+2. Verify the shard is `ACTIVE`.
+3. Increment `writer_epoch`.
+4. Atomically write a new manifest version.
+5. If the manifest write loses a race, reload and retry.
 
-Before any manifest update, a writer must verify its `writer_epoch` remains valid:
+Before a manifest update, a writer verifies its local epoch is still current:
 
-- If `local_writer_epoch == stored_writer_epoch`: The writer is still active and may proceed
-- If `local_writer_epoch < stored_writer_epoch`: The writer has been fenced and must abort
+- If `local_writer_epoch == stored_writer_epoch`, the writer may proceed.
+- If `local_writer_epoch < stored_writer_epoch`, the writer has been fenced and must abort.
 
-For a concrete example, see [Appendix 1: Writer Fencing Example](#appendix-1-writer-fencing-example).
+WAL append conflicts also detect fencing.
+If an older writer collides with a newer writer's WAL entry at the same position, it reloads the manifest and observes the higher epoch.
+Fence sentinel entries make this collision path explicit without storing data batches.
 
 ## Background Job Expectations
 
-Background jobs handle merging flushed MemTables to the base table and garbage collection.
+Background jobs compact SSTables into the base table and remove obsolete shard data.
 
-### MemTable Merger
+### SSTable Compactor
 
-Flushed MemTables must be merged to the base table in **ascending generation order** within each shard. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones.
+SSTables must be compacted into the base table in ascending generation order within each shard.
+The compaction uses Lance merge-insert semantics and updates `compacted_sstables[shard_id]` atomically with the base-table commit.
 
-The merge uses Lance's merge-insert operation with atomic transaction semantics:
+On commit conflict, a compactor reloads the conflicting base-table version:
 
-- `merged_generations[shard_id]` is updated atomically with the data commit
-- On commit conflict, check the conflicting commit's `merged_generations` to determine if the generation was already merged
-
-For a concrete example, see [Appendix 2: Concurrent Merger Example](#appendix-2-concurrent-merger-example).
+- If the committed `compacted_sstables[shard_id]` is already greater than or equal to the generation being compacted, the compactor skips that generation.
+- Otherwise, the compactor retries from the latest base-table version.
 
 ### Garbage Collector
 
-The garbage collector removes obsolete data from shard directories. Flushed MemTables and their referenced WAL files may be deleted after:
-
-1. The generation has been merged to the base table (`generation <= merged_generations[shard_id]`)
-2. All maintained indexes have caught up (`generation <= min(index_catchup[I].caught_up_generation)`)
-3. No retained base table version references the generation for time travel
+The garbage collector may remove obsolete SSTables after:
 
-!!!warning
-    Deleting WAL files weakens [writer fencing](#writer-fencing) and can lead to silent acknowledgement of lost writes.
+1. The SSTable has been compacted into the base table.
+2. Every maintained index has caught up to cover the SSTable's generation, or the SSTable is no longer needed for indexed reads.
+3. No retained base-table version needs the SSTable for time travel or consistency.
 
-    Fencing detects a stalled writer when its `put-if-not-exists` for the next WAL entry collides with a newer writer's entry at the same position — only that collision triggers the epoch check. If GC has already removed the WAL file at that position, the stalled writer's PUT lands on empty space and succeeds against its old `writer_epoch`. The entry is acknowledged to the client, but the new manifest's `replay_after_wal_entry_position` has already advanced past it, so the data is never replayed.
+!!! warning
+    Deleting WAL files can weaken writer fencing.
 
-    Implementations that GC WAL files must compensate, for example by re-checking fence state after each successful WAL write, encoding the writer epoch into the WAL filename so positions are partitioned by epoch, or otherwise guaranteeing a stalled writer cannot land at a position that has been or will be GC'd.
+    Fencing detects a stalled writer when its put-if-not-exists for the next WAL entry collides with a newer writer's entry at the same position.
+    If garbage collection has removed that WAL file, the stalled writer may write into empty space with an old `writer_epoch`.
+    Implementations that garbage collect WAL files must compensate by re-checking fence state after WAL writes, partitioning WAL positions by epoch, or otherwise preventing stale writers from landing at positions that have been garbage collected.
 
 ## Reader Expectations
 
 ### LSM Tree Merging Read
 
-For tables with a primary key, readers **MUST** merge results from multiple data sources
-(base table, flushed MemTables, in-memory MemTables) by primary key to ensure correctness.
-
-When the same primary key exists in multiple sources, the reader must keep only the newest version based on:
-
-1. **Generation number** (`_gen`): Higher generation wins. The base table has generation 0, MemTables have positive integers starting from 1.
-2. **Row address** (`_rowaddr`): Within the same generation, higher row address wins (later writes within a batch overwrite earlier ones).
-
-The ordering for "newest" is: highest `_gen` first, then highest `_rowaddr`.
+For primary-key tables, readers merge rows from the base table, SSTables, and optionally in-memory MemTables by primary key.
+The newest row wins.
 
-This deduplication is essential because:
+Freshness ordering within one shard is:
 
-- A row updated in a MemTable also exists (with older data) in the base table
-- A flushed MemTable that has been merged to the base table may not yet be garbage collected, causing the same row to appear in both
-- A single write batch may contain multiple updates to the same primary key
+1. The active MemTable wins over every SSTable and the base table.
+2. Among SSTables, higher generation wins.
+3. Any uncompacted SSTable wins over the base table.
+4. Within the active MemTable, higher row position wins.
+5. Within an SSTable, its deletion vector has already hidden older duplicate primary-key rows.
 
-Without proper merging, queries would return duplicate or stale rows.
+For freshness comparisons, the base table uses the sentinel generation 0.
+SSTable generations are positive.
+This ordering applies only to sources selected for the same read plan.
+Readers must not include an SSTable that is already covered by the base table according to `compacted_sstables[shard_id]`, because otherwise the positive SSTable generation would incorrectly outrank base-table rows during deduplication.
+Rows from different shards do not need primary-key deduplication if the sharding spec guarantees that each primary key maps to exactly one shard.
 
 Append-only tables without a primary key do not perform primary-key deduplication.
-Readers should include the relevant base table, flushed MemTables, and in-memory MemTables
-according to the requested consistency level; duplicate values are treated as distinct appended rows.
+Rows from all selected sources are distinct appended rows.
 
-### Reader Consistency
-
-Reader consistency depends on two factors:
+### Tombstones
 
-1. access to in-memory MemTables
-2. the source of shard metadata (either through MemWAL index or shard manifests)
+Readers must treat `_tombstone = true` rows as delete markers.
+In SSTables, deletion vectors first resolve same-generation duplicate primary keys.
+Then query planning filters tombstone rows from user-visible results.
+In active in-memory MemTables, the newest visible row position for a primary key wins; if that row is a tombstone, the key is absent.
 
-Strong consistency requires access to in-memory MemTables for all shards involved in the query and reading shard manifests directly.
-Otherwise, the query is eventually consistent due to missing unflushed data or stale MemWAL Index snapshots.
-
-!!!note
-Reading a stale MemWAL Index does not impact correctness, only freshness:
+### Reader Consistency
 
-    - **Merged MemTable still in index**: If a flushed MemTable has been merged to the base table but still shows in the MemWAL index, readers query both. This results in some inefficiency for querying the same data twice, but [LSM-tree merging](#lsm-tree-merging-read) ensures correct results since both contain the same data. The inefficiency is also compensated by the fact that the data is covered by index and we rarely end up scanning both data.
-    - **Garbage collected MemTable still in index**: If a flushed MemTable has been garbage collected, but is still in the MemWAL index, readers would fail to open it and skip it. This is also safe because if it is garbage collected, the data must already exist in the base table.
-    - **Newly flushed MemTable not in index**: If a newly flushed MemTable is added after the snapshot was built, it is not queried. The result is eventually consistent but correct for the snapshot's point in time.
+Reader consistency depends on:
 
-### Query Planning
+1. Whether the reader can access active in-memory MemTables.
+2. Whether shard metadata comes from latest shard manifests or from an older MemWAL index snapshot.
 
-#### MemTable Collection
+Strong consistency requires active in-memory MemTable access for relevant shards and direct reads of latest shard manifests.
+Otherwise, reads are eventually consistent because unflushed data or newly-created shards may be absent from the read plan.
 
-The query planner collects datasets from multiple sources and assembles them for unified query execution.
-Datasets come from:
+Reading a stale MemWAL index snapshot does not corrupt last-write-wins ordering, but it can reduce freshness:
 
-1. base table (representing already-merged data)
-2. flushed MemTables (persisted but not yet merged)
-3. optionally in-memory MemTables (if accessible).
+- If a compacted SSTable is still listed, readers must skip it when `generation <= compacted_sstables[shard_id]`.
+    For primary-key tables, including it would let an older SSTable row outrank newer base-table contents because SSTable generations are positive and the base table is modeled as generation 0.
+    For append-only tables, including it would return the same append twice.
+- If a garbage-collected SSTable is still listed, readers may skip it after failing to open it because its data must already be in the base table or be filtered out by `compacted_sstables`.
+- If a new SSTable is not listed, the read is consistent with the older snapshot but may miss fresher data.
 
-Each dataset is tagged with a generation number: 0 for the base table, and positive integers for MemTable generations.
-Within a shard, the generation number determines data freshness, with higher numbers representing newer data.
-For primary-key tables, rows from different shards do not need deduplication
-since each primary key maps to exactly one shard.
-Append-only tables without a primary key do not require cross-shard primary-key deduplication.
+Readers that require latest shard membership should list `_mem_wal/` and read shard manifests instead of relying only on snapshots.
 
-The planner also collects bloom filters from each generation for staleness detection during search queries.
+### Query Planning
 
-#### Shard Pruning
+A query planner collects sources from:
 
-Before executing queries, if sharding spec is available,
-the planner evaluates filter predicates against sharding specs to determine which shards may contain matching data.
-This pruning step reduces the number of shards to scan.
+1. The base table.
+2. SSTables that are not yet safely replaceable by base-table indexed reads.
+3. Active in-memory MemTables, when available and required by the requested consistency level.
 
-For each filter predicate:
+Each source is tagged with its shard and freshness tier.
+SSTable sources are also tagged with their generation.
+For primary-key reads, the planner applies LSM deduplication across selected sources.
+For append-only reads, the planner concatenates selected sources without primary-key deduplication.
 
-1. Extract predicates on columns used in sharding specs
-2. Evaluate which shard values can satisfy the predicate
-3. Prune shards whose values cannot match
+Bloom filters and `_pk_index/` sidecars help prune SSTables during point lookups and cross-generation deduplication.
 
-For example, with a sharding spec using `bucket(user_id, 10)` and a filter `user_id = 123`:
+### Shard Pruning
 
-1. Compute `bucket(123, 10) = 3`
-2. Only scan shards with bucket value 3
-3. Skip all other shards
+When sharding specs are available, the planner evaluates query predicates against shard fields and skips shards whose computed shard values cannot match.
 
-Shard pruning applies to both scan queries and prefilters in search queries.
+For example, with `bucket(user_id, 10)` and predicate `user_id = 123`:
 
-#### Indexed Read Plan
+1. Compute the bucket id for `123`.
+2. Scan only shards whose manifest has the same computed bucket value.
+3. Skip all other bucket shards.
 
-When data is merged from a flushed MemTable to the base table, the base table's indexes are rebuilt asynchronously by the base table index builders.
-During this window, the merged data exists in the base table but is not yet covered by the base table's indexes.
+### Indexed Read Plan
 
-Without special handling, indexed queries would fall back to expensive full scans for the unindexed part of the base table.
-To maintain indexed read performance, the query planner should use `index_catchup` progress to determine the optimal data source for each query.
+When data is compacted from an SSTable into the base table, base-table indexes may lag behind the data commit.
+`index_catchup` records which compacted generation each base-table index covers.
 
-The key insight is that flushed MemTables serve as a bridge between the base table's index catchup and the current merged state.
-For a query that requires a specific index for acceleration, when `index_gen < merged_gen`,
-the generations in the gap `(index_gen, merged_gen]` have data already merged in the base table but are not covered by the base table's index.
-Since flushed MemTables contain pre-built indexes (created during [MemTable flush](#flushed-memtable)), queries can use these indexes instead of scanning unindexed data in the base table.
-This ensures all reads remain indexed regardless of how far behind the async index builder is.
+If an indexed query needs index `I` and `I` has only caught up to generation `G` while `compacted_sstables[shard_id]` is higher, the planner should read the gap from SSTable indexes instead of scanning unindexed base-table rows.
+Once index `I` catches up, the planner can use the base-table index for those compacted rows.
 
 ## Appendices
 
 ### Appendix 1: Writer Fencing Example
 
-This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same shard.
-
-#### Initial State
+Initial shard manifest:
 
+```text
+version: 1
+writer_epoch: 5
+replay_after_wal_entry_position: 10
+wal_entry_position_last_seen: 12
+status: ACTIVE
 ```
-Shard manifest (version 1):
-  writer_epoch: 5
-  replay_after_wal_entry_position: 10
-  wal_entry_position_last_seen: 12
-```
-
-#### Scenario
-
-| Step | Writer A                                      | Writer B                                  | Manifest State     |
-| ---- | --------------------------------------------- | ----------------------------------------- | ------------------ |
-| 1    | Loads manifest, sees epoch=5                  |                                           | epoch=5, version=1 |
-| 2    | Increments to epoch=6, writes manifest v2     |                                           | epoch=6, version=2 |
-| 3    | Starts writing WAL entries 13, 14, 15         |                                           |                    |
-| 4    |                                               | Loads manifest v2, sees epoch=6           | epoch=6, version=2 |
-| 5    |                                               | Increments to epoch=7, writes manifest v3 | epoch=7, version=3 |
-| 6    |                                               | Starts writing WAL entries 16, 17         |                    |
-| 7    | Tries to flush MemTable, loads manifest       |                                           |                    |
-| 8    | Sees epoch=7, but local epoch=6               |                                           |                    |
-| 9    | **Writer A is fenced!** Aborts all operations |                                           |                    |
-| 10   |                                               | Continues writing normally                | epoch=7, version=3 |
-
-#### What Happens to Writer A's WAL Entries?
-
-Writer A wrote WAL entries 13, 14, 15 with `writer_epoch=6` in their schema metadata.
-
-When Writer B performs crash recovery or MemTable flush:
-
-1. Reads WAL entries sequentially starting from `replay_after_wal_entry_position + 1` (entry 11, since positions are 0-based)
-2. For each entry, checks existence using HEAD request on the bit-reversed filename
-3. Continues until an entry is not found (e.g., entry 18 doesn't exist)
-4. Finds entries 13, 14, 15, 16, 17
-5. Reads each file's `writer_epoch` from schema metadata
-6. Entries 13, 14, 15 have `writer_epoch=6` which is <= current epoch (7) -> **valid, will be replayed**
-7. Entries 16, 17 have `writer_epoch=7` -> **valid, will be replayed**
 
-#### Key Points
+Writer A loads version 1, claims epoch 6, and writes manifest version 2.
+It appends WAL entries 13, 14, and 15 with `writer_epoch = 6`.
 
-1. **No data loss**: Writer A's entries are not discarded. They were written with a valid epoch at the time and will be included in recovery.
+Writer B then loads version 2, claims epoch 7, and writes manifest version 3.
+It appends WAL entries 16 and 17 with `writer_epoch = 7`.
 
-2. **Consistency preserved**: Writer A is prevented from making further writes that could conflict with Writer B.
+When Writer A later tries to flush or update the shard manifest, it reloads the manifest and sees stored epoch 7 while its local epoch is 6.
+Writer A is fenced and must abort.
 
-3. **Orphaned files are safe**: WAL files from fenced writers remain on storage and are replayed by the new writer. They are only garbage collected after being included in a flushed MemTable that has been merged.
+Recovery starts from `replay_after_wal_entry_position + 1`, which is entry 11.
+Entries 13, 14, 15, 16, and 17 are valid replay inputs because they were written by epochs that were valid at write time and are not greater than the current shard epoch.
 
-4. **Epoch validation timing**: Writers check their epoch before manifest updates (MemTable flush), not on every WAL write. This keeps the hot path fast while ensuring consistency at commit boundaries.
+### Appendix 2: Concurrent Compactor Example
 
-### Appendix 2: Concurrent Merger Example
+Initial state:
 
-This example demonstrates how MemWAL Index and conflict resolution handle concurrent mergers safely.
+```text
+MemWAL index:
+  compacted_sstables: {shard: 5}
 
-#### Initial State
-
-```
-MemWAL Index:
-  merged_generations: {shard: 5}
-
-Shard manifest (version 1):
+Shard manifest:
   current_generation: 8
-  flushed_generations: [(6, "abc123_gen_6"), (7, "def456_gen_7")]
+  sstables:
+    - generation: 6, path: "abc12345_gen_6"
+    - generation: 7, path: "def67890_gen_7"
 ```
 
-#### Scenario 1: Racing on the Same Generation
-
-Two mergers both try to merge generation 6 concurrently.
-
-| Step | Merger A                  | Merger B                       | MemWAL Index     |
-| ---- | ------------------------- | ------------------------------ | ---------------- |
-| 1    | Reads index: merged_gen=5 |                                | merged_gen=5     |
-| 2    | Reads shard manifest     |                                |                  |
-| 3    | Starts merging gen 6      |                                |                  |
-| 4    |                           | Reads index: merged_gen=5      | merged_gen=5     |
-| 5    |                           | Reads shard manifest          |                  |
-| 6    |                           | Starts merging gen 6           |                  |
-| 7    | Commits (merged_gen=6)    |                                | **merged_gen=6** |
-| 8    |                           | Tries to commit                |                  |
-| 9    |                           | **Conflict**: reads new index  |                  |
-| 10   |                           | Sees merged_gen=6 >= 6, aborts |                  |
-| 11   |                           | Reloads, continues to gen 7    |                  |
+Two compactors both try to compact the SSTable at generation 6.
+Compactor A commits first and updates `compacted_sstables[shard]` to 6 in the same base-table commit as the data.
+Compactor B then hits a commit conflict, reloads the latest MemWAL index, sees `compacted_sstables[shard] >= 6`, skips generation 6, and continues with generation 7.
 
-Merger B's conflict resolution detected that generation 6 was already merged by checking the MemWAL Index in the conflicting commit.
+The MemWAL index is the authoritative compaction-progress record because it is committed atomically with the base-table data changes.
 
-#### Scenario 2: Crash After Table Commit
+### Appendix 3: Bucket Hashing
 
-Merger A crashes after committing to the table.
+The bucket transform hash uses 32-bit wrapping arithmetic with these mixing functions.
+Right shifts in `fmix` are logical shifts of the `u32` bit pattern.
 
-| Step | Merger A                  | Merger B                         | MemWAL Index     |
-| ---- | ------------------------- | -------------------------------- | ---------------- |
-| 1    | Reads index: merged_gen=5 |                                  | merged_gen=5     |
-| 2    | Merges gen 6, commits     |                                  | **merged_gen=6** |
-| 3    | **CRASH**                 |                                  | merged_gen=6     |
-| 4    |                           | Reads index: merged_gen=6        | merged_gen=6     |
-| 5    |                           | Reads shard manifest            |                  |
-| 6    |                           | **Skips gen 6** (already merged) |                  |
-| 7    |                           | Merges gen 7, commits            | **merged_gen=7** |
-
-The MemWAL Index is the single source of truth. Merger B correctly used it to determine that generation 6 was already merged.
-
-#### Key Points
+```text
+mix_k1(k) = rotl32(k * 0xcc9e2d51, 15) * 0x1b873593
+mix_h1(h, k) = rotl32(h ^ k, 13) * 5 + 0xe6546b64
+fmix(h, len) =
+    h = h ^ len
+    h = (h ^ (h >> 16)) * 0x85ebca6b
+    h = (h ^ (h >> 13)) * 0xc2b2ae35
+    h ^ (h >> 16)
+```
 
-1. **Single source of truth**: `merged_generations` is the authoritative source for merge progress, updated atomically with data.
+Signed and unsigned casts use two's-complement wrapping.
+Values are normalized and hashed as follows:
+
+- `bool`: `false` as `0`, `true` as `1`, then `hash_i32`.
+- `int8`, `int16`, `int32`, `uint8`, `uint16`, `uint32`, `date32`, `time32`: cast to `i32`, then `hash_i32`.
+- `int64`, `uint64`, `timestamp`, `time64`: cast to `i64`, then `hash_i64`.
+- `float32`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7fc00000`; other values use IEEE 754 bits cast to `i32`, then `hash_i32`.
+- `float64`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7ff8000000000000`; other values use IEEE 754 bits cast to `i64`, then `hash_i64`.
+- `utf8` and `large_utf8`: hash the UTF-8 bytes with `hash_bytes`.
+
+The helper hashes are:
+
+```text
+hash_i32(v) = fmix(mix_h1(0, mix_k1(v)), 4)
+
+hash_i64(v) =
+    low = low 32 bits of v as i32
+    high = high 32 bits of v as i32
+    fmix(mix_h1(mix_h1(0, mix_k1(low)), mix_k1(high)), 8)
+
+hash_bytes(bytes) =
+    h = 0
+    for each complete 4-byte little-endian chunk:
+        h = mix_h1(h, mix_k1(chunk_as_i32))
+    for each remaining byte:
+        h = mix_h1(h, mix_k1(sign_extend_i8(byte)))
+    fmix(h, byte_length)
+```
 
-2. **Conflict resolution uses MemWAL Index**: When a commit conflicts, the merger checks the conflicting commit's MemWAL Index.
+Test vectors for `num_buckets = 8`:
 
-3. **No progress regression**: Because MemWAL Index is updated atomically with data, concurrent mergers cannot regress the merge progress.
+- `int32` or `date32`: `1 -> 2`, `2 -> 7`, `null -> 0`, `3 -> 1`.
+- `utf8`: `"a" -> 1`, `"b" -> 5`, `null -> 0`.
+- `bool`: `true -> 2`.
+- `float32`: `1.25 -> 0`.
+- `float64`: `1.25 -> 0`.
diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md
index 78dd5301fb8..ef3384c6254 100644
--- a/docs/src/format/table/transaction.md
+++ b/docs/src/format/table/transaction.md
@@ -466,6 +466,64 @@ The following operations are retryable conflicts with DataReplacement:
 A concurrent Delete or Update that only adds a deletion vector to a target fragment (without removing it) is compatible: the
 positional column file stays aligned and the rebase preserves the deletion vector.
 
+### DataOverlay
+
+Attaches [overlay files](data_overlay_file.md) to fragments, supplying new values
+for a subset of `(row offset, field)` cells without rewriting the fragments' base
+data files. The overlays are appended to each fragment's existing `overlays` list,
+so overlays written by concurrent commits are preserved. Each overlay's
+`committed_version` is stamped to the new dataset version at commit time (and
+re-stamped on retry), like the created-at / last-updated-at version sequences.
+
+<details>
+<summary>DataOverlay protobuf message</summary>
+
+```protobuf
+%%% proto.message.DataOverlay %%%
+
+%%% proto.message.DataOverlayGroup %%%
+```
+
+</details>
+
+#### DataOverlay Compatibility
+
+A DataOverlay operation only changes cells within existing fragments and preserves
+physical row addresses, so — like DataReplacement — it is intentionally permissive.
+Because overlays stack and the higher `committed_version` wins each covered cell,
+independent backfills never conflict, and a concurrent Delete simply makes the
+overlay value for a deleted offset inert. Here are the operations that conflict
+with DataOverlay:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+The following operations are retryable conflicts with DataOverlay:
+
+- Rewrite (only if overlapping fragments) — row-rewriting compaction or an
+  overlay→base fold changes physical row addresses or consumes the overlays, so
+  the overlay's offsets are no longer valid; the writer must re-read the new
+  fragment, recompute, and retry.
+- Merge (always).
+- A row-moving Update that touches an overlaid fragment — a delete-and-reinsert
+  update (any update that is not a `REWRITE_COLUMNS` column rewrite) relocates the
+  updated rows into new fragments, so the overlay's physical offsets no longer
+  address them; the writer must re-read and retry.
+
+DataOverlay is compatible with another DataOverlay (any fields), Append, Delete, a
+`REWRITE_COLUMNS` column rewrite, and DataReplacement, because all of these
+preserve physical row addresses: overlay offsets stay valid, the overlay is newer
+and wins its covered cells, and the version gate excludes those cells from any
+rebuilt index.
+
+When a DataReplacement or a `REWRITE_COLUMNS` update writes new base values for a
+field, it supersedes any older overlay on that field: the writer tombstones the
+overlay's entry for the rewritten field — replacing the field id with the obsolete
+sentinel, as with obsolete base columns — so the fresh base values are not silently
+shadowed. Overlay entries for other fields are preserved, and an overlay left with
+no live fields is dropped.
+
 ### UpdateMemWalState
 
 Updates the state of MemWal indices (write-ahead log based indices).
diff --git a/docs/src/guide/.pages b/docs/src/guide/.pages
index 46ddd475799..7a0fd817b1c 100644
--- a/docs/src/guide/.pages
+++ b/docs/src/guide/.pages
@@ -6,6 +6,7 @@ nav:
   - JSON Support: json.md
   - Tags and Branches: tags_and_branches.md
   - Object Store Configuration: object_store.md
+  - Observability: observability.md
   - Distributed Write: distributed_write.md
   - Distributed Indexing: distributed_indexing.md
   - Migration Guide: migration.md
diff --git a/docs/src/guide/arrays.md b/docs/src/guide/arrays.md
index f824dd641c1..5765ca7675b 100644
--- a/docs/src/guide/arrays.md
+++ b/docs/src/guide/arrays.md
@@ -119,11 +119,11 @@ calling `lance.arrow.ImageTensorArray.to_encoded`.
 
 A `lance.arrow.EncodedImageArray.to_tensor` method is provided to decode
 encoded images and return them as `lance.arrow.FixedShapeImageTensorArray`, from
-which they can be converted to numpy arrays or TensorFlow tensors.
+which they can be converted to numpy arrays.
 For decoding images, it will first attempt to use a decoder provided via the optional
 function parameter. If decoder is not provided it will attempt to use
-[Pillow](https://pillow.readthedocs.io/en/stable/) and [tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) in that
-order. If neither library or custom decoder is available an exception will be raised.
+[Pillow](https://pillow.readthedocs.io/en/stable/). If neither Pillow nor a custom
+decoder is available an exception will be raised.
 
 ```python
 from lance.arrow import ImageURIArray
@@ -132,13 +132,16 @@ uris = [os.path.join(os.path.dirname(__file__), "images/1.png")]
 encoded_images = ImageURIArray.from_uris(uris).read_uris()
 print(encoded_images.to_tensor())
 
-def tensorflow_decoder(images):
-    import tensorflow as tf
+def pillow_decoder(images):
+    import io
     import numpy as np
+    from PIL import Image
 
-    return np.stack(tf.io.decode_png(img.as_py(), channels=3) for img in images.storage)
+    return np.stack(
+        np.asarray(Image.open(io.BytesIO(img.as_py()))) for img in images.storage
+    )
 
-print(encoded_images.to_tensor(tensorflow_decoder))
+print(encoded_images.to_tensor(pillow_decoder))
 # <lance.arrow.FixedShapeImageTensorArray object at 0x...>
 # [[42, 42, 42, 255]]
 # <lance.arrow.FixedShapeImageTensorArray object at 0x...>
@@ -164,8 +167,8 @@ created by calling `lance.arrow.ImageArray.from_array` and passing in a
 It can be encoded into to `lance.arrow.EncodedImageArray` by calling
 `lance.arrow.FixedShapeImageTensorArray.to_encoded` and passing custom encoder
 If encoder is not provided it will attempt to use
-[tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) and [Pillow](https://pillow.readthedocs.io/en/stable/) in that order. Default encoders will
-encode to PNG. If neither library is available it will raise an exception.
+[Pillow](https://pillow.readthedocs.io/en/stable/). The default encoder will encode
+to PNG. If neither Pillow nor a custom encoder is available it will raise an exception.
 
 ```python
 from lance.arrow import ImageURIArray
@@ -176,4 +179,4 @@ tensor_images.to_encoded()
 # <lance.arrow.EncodedImageArray object at 0x...>
 # [...
 # b'\x89PNG\r\n\x1a...'
-``` 
\ No newline at end of file
+```
diff --git a/docs/src/guide/blob.md b/docs/src/guide/blob.md
index 00bd5d086e3..0d0d4002040 100644
--- a/docs/src/guide/blob.md
+++ b/docs/src/guide/blob.md
@@ -174,8 +174,9 @@ Choose the read API based on the payload shape you want:
 
 | API | Returns | Use When |
 |---|---|---|
-| `read_blobs` | `List[Tuple[int, bytes]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. |
-| `take_blobs` | `List[BlobFile]` | You need file-like objects for streaming, seeking, or partial reads. |
+| `read_blobs` | `List[Tuple[int, Optional[bytes]]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. |
+| `read_blob_ranges` | `List[Tuple[int, int, Optional[bytes]]]` | You need selected byte ranges from multiple rows without materializing complete blobs. |
+| `take_blobs` | `List[Optional[BlobFile]]` | You need file-like objects for streaming, seeking, or partial reads. |
 | `scanner(..., blob_handling="all_binary")` | Arrow binary columns | You want blob columns in a scan result or `pyarrow.Table`. |
 
 Do not wrap `take_blobs` in your own thread pool just to call `read()` or
@@ -183,7 +184,8 @@ Do not wrap `take_blobs` in your own thread pool just to call `read()` or
 batched blob reads through Lance's scheduler.
 
 Exactly one selector must be provided to `read_blobs` or `take_blobs`: `ids`,
-`indices`, or `addresses`.
+`indices`, or `addresses`. `read_blob_ranges` accepts the same selector kinds
+through its required `selector` argument.
 
 | Selector | Typical Use | Stability |
 |---|---|---|
@@ -223,6 +225,48 @@ row_addrs = ds.to_table(columns=[], with_row_address=True).column("_rowaddr").to
 rows = ds.read_blobs("blob", addresses=row_addrs[:2])
 ```
 
+Blob selection APIs preserve logical result cardinality. `read_blobs()` and
+`take_blobs()` return one element per selected row, and `read_blob_ranges()`
+returns one element per request. A null blob is returned as `None`; a valid
+empty blob remains a non-null empty payload or zero-length `BlobFile`.
+
+### Read row-specific byte ranges
+
+Use `read_blob_ranges` to read multiple blob-local ranges with one planned API
+call. Each request is a `(row, offset, length)` tuple, and `selector` determines
+whether every `row` is interpreted as a row ID, row address, or dataset index.
+
+```python
+import lance
+
+ds = lance.dataset("./blobs_v22.lance")
+results = ds.read_blob_ranges(
+    "blob",
+    requests=[
+        (7, 0, 1024),
+        (7, 4096, 1024),
+        (12, 0, 0),
+    ],
+    selector="indices",
+)
+
+for request_index, row_address, data in results:
+    if data is None:
+        # The selected blob is null.
+        continue
+    print(request_index, row_address, len(data))
+```
+
+Each result contains the zero-based `request_index`, the resolved physical row
+address, and the requested bytes. `request_index` identifies the original
+request when the same row appears more than once.
+
+A request on a null blob returns `None`, including when its range is empty. An
+empty range on a non-null blob returns `b""` without payload I/O. For every
+request, `offset + length` must fit in an unsigned 64-bit integer. A range on a
+non-null blob must not extend beyond its logical size; blob-local bounds are not
+evaluated for null blobs because they have no logical payload length.
+
 ### Read blob columns as Arrow binary
 
 ```python
@@ -241,8 +285,10 @@ import lance
 ds = lance.dataset("./blobs_v22.lance")
 blobs = ds.take_blobs("blob", indices=[0, 1])
 
-with blobs[0] as f:
-    header = f.read(1024)
+blob = blobs[0]
+if blob is not None:
+    with blob as f:
+        header = f.read(1024)
 ```
 
 ### Example: decode video frames lazily
@@ -253,6 +299,8 @@ import lance
 
 ds = lance.dataset("./videos_v22.lance")
 blob = ds.take_blobs("video", indices=[0])[0]
+if blob is None:
+    raise ValueError("video blob is null")
 
 start_ms, end_ms = 500, 1000
 
@@ -347,7 +395,7 @@ lance.write_dataset(
 
 Not every binary column needs to be a blob column. Plain Arrow `binary`/`large_binary` stores bytes *inline*, interleaved with your other columns, which is simplest and fastest for really small blobs (e.g., thumbnail images). Using a blob column to store the binary payload makes sense when either of these holds:
 
-- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `take_blobs` → `BlobFile` handles that seek and range-read, so you pay only for the bytes you touch.
+- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `read_blob_ranges` for planned row-specific range reads and `take_blobs` → `BlobFile` handles for caller-driven seeks, so you pay only for the bytes you touch.
 - **Your values are large (roughly 1 MB or more on average).** Operations that rewrite entire rows, such as compaction or some updates, must copy the large inline payloads forward into the new version — even when those bytes never changed. The bigger the payload, the more bytes you rewrite per logical change (write amplification). A blob column keeps large payloads in separate `.blob` files that are referenced rather than re-copied, so these operations don't rewrite the heavy bytes.
 
 !!! tip
diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md
index 5efd7b26b7f..3222326bea6 100644
--- a/docs/src/guide/migration.md
+++ b/docs/src/guide/migration.md
@@ -8,24 +8,18 @@ migrate.
 
 ## 9.0.0
 
-* Newly created FTS / inverted indexes now default to format v2 instead of v1.
-  The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the
-  format used for newly created indexes. Users who need a specific index layout
-  should pass the index creation parameter `format_version` explicitly.
-
-* This affects users who create FTS / inverted indexes and need those indexes to
-  be readable by older Lance versions, or who depend on the v1 index layout. In
-  those cases, pass `format_version=1` when creating the index. Otherwise, newly
-  created indexes will use v2 by default, and older Lance readers may not be able
-  to read them.
-
-  ```python
-  dataset.create_scalar_index("text", "INVERTED", format_version=1)
-  ```
-
-* Existing v1 FTS indexes remain queryable. Operations that maintain an existing
-  v1 index, including append, incremental indexing, optimize, and mem-wal
-  maintained-index flush, should continue preserving the v1 format.
+* Unless overridden, newly created FTS indexes use format v2. The code analyzer
+  and `block_size=256` require format v3, so readers must support v3 before an
+  index using either option is created.
+
+* To keep new indexes readable by nodes that support at most format v1 or v2,
+  set `format_version` in the index creation parameters, or set
+  `LANCE_FTS_FORMAT_VERSION` for a rollout-wide override. Formats v1 and v2
+  require the text analyzer and `block_size=128`.
+
+* Operations that maintain an existing FTS index, including append, incremental
+  indexing, optimize, and mem-wal maintained-index flush, preserve its format
+  version.
 
 ## 7.2.0
 
diff --git a/docs/src/guide/observability.md b/docs/src/guide/observability.md
new file mode 100644
index 00000000000..f2053b59447
--- /dev/null
+++ b/docs/src/guide/observability.md
@@ -0,0 +1,69 @@
+# Observability
+
+Lance can publish operational metrics to your monitoring stack. The table below
+is the authoritative catalogue of the metrics Lance emits, shared verbatim with
+the Rust [`lance::metrics`](https://docs.rs/lance/latest/lance/metrics/) module
+documentation.
+
+--8<-- "rust/lance/src/metrics.md"
+
+## Collecting metrics
+
+Lance emits through the [`metrics`](https://docs.rs/metrics) crate facade, so it
+is not tied to a specific backend — you install a recorder/exporter and route
+the metrics wherever you like. Metrics are available from **both** the Rust and
+Python APIs.
+
+### Rust
+
+Enable the `metrics` feature on the `lance` crate:
+
+```toml
+lance = { version = "...", features = ["metrics"] }
+```
+
+Then install any `metrics`-compatible recorder once at startup, before opening
+datasets. For example, with
+[`metrics-exporter-prometheus`](https://docs.rs/metrics-exporter-prometheus):
+
+```rust
+metrics_exporter_prometheus::PrometheusBuilder::new()
+    .install()
+    .expect("install Prometheus recorder");
+```
+
+Any recorder works — Prometheus, StatsD, an OpenTelemetry bridge, and so on.
+When no recorder is installed, emission is a cheap no-op.
+
+### Python
+
+Unlike Rust, the Python bindings do not let you plug in an arbitrary recorder:
+bridging one across the FFI boundary into the Rust `metrics` facade would be
+complicated and inefficient. Instead `pylance` standardizes on OpenTelemetry,
+which has good Python support, as its recorder.
+
+The `pylance` wheels are built with the `metrics` feature enabled. Install the
+OpenTelemetry extra and call `instrument_lance_metrics`, which registers Lance's
+metrics as observable instruments on your OpenTelemetry `MeterProvider`:
+
+```bash
+pip install "pylance[otel]"
+```
+
+```python
+from lance.otel import instrument_lance_metrics
+
+# Uses the global MeterProvider; pass meter_provider=... to target a specific one.
+instrument_lance_metrics()
+```
+
+From there the metrics flow through whatever OpenTelemetry pipeline you have
+configured (OTLP, Prometheus, console, …). Because OpenTelemetry has no
+asynchronous histogram instrument, histograms are exported Prometheus-style as
+three observable counters: `<name>_bucket`, `<name>_count`, and `<name>_sum`.
+Each `<name>_bucket` sample carries an `le` ("less than or equal") attribute
+giving that bucket's inclusive upper bound in the metric's unit; the bucket
+count is cumulative, covering every observation at or below `le`. For example, a
+`lance_object_store_request_duration_seconds_bucket` sample with `le="0.5"`
+counts all requests that completed in 0.5 seconds or less, while `le="+Inf"` is
+the total count.
diff --git a/docs/src/guide/performance.md b/docs/src/guide/performance.md
index 14181eb69af..acccdf14776 100644
--- a/docs/src/guide/performance.md
+++ b/docs/src/guide/performance.md
@@ -219,6 +219,37 @@ use cases. For example, S3 can typically get up to 5000
 req/s and with these settings we should get there in about
 10 seconds.
 
+## Fragment Sizing
+
+A Lance table is a collection of fragments tracked by a manifest. How you size those fragments
+trades off two classes of work:
+
+- **Manifest-level operations** scale with the *number* of fragments. Every dataset mutation
+  (appends, metadata updates, schema changes, compactions, etc.) rewrites the manifest, so a
+  larger fragment list makes every write slower. Reads pay a similar cost up front: opening a
+  dataset, listing fragments, planning a scan, and resolving transaction conflicts at the
+  dataset level all walk the manifest.
+- **Fragment-level operations** scale with the *size* of a fragment. These include scans
+  against a matching fragment, compaction, updates, deletes, and `merge_insert`. Conflict
+  detection for these operations is also done at the fragment level.
+
+Fewer, larger fragments make manifest-level operations cheap but make each fragment-level
+operation heavier and increase the chance of conflicts when many writers target the same
+fragment. More, smaller fragments do the reverse.
+
+Practical guidance:
+
+- The default of 1M rows per fragment works well up to ~1B rows. Past that, bumping toward
+  ~100M rows per fragment is reasonable, though fragment-count limits are rarely the bottleneck
+  in practice.
+- Tens of thousands of fragments per table is generally fine.
+- Keep individual fragments well under object-store object-size limits (S3 caps at 5 TB, and
+  stores tend to misbehave well before that). 10 GB–100 GB per fragment is a reasonable upper
+  range; 1 TB is a hard ceiling.
+- If you run many concurrent updates, deletes, or `merge_insert` operations, err toward more
+  fragments — conflict detection is per-fragment, so too few fragments leads to excess
+  retries.
+
 ## Conflict Handling
 
 Lance supports concurrent operations on the same table using optimistic concurrency control. When two
diff --git a/docs/src/guide/read_and_write.md b/docs/src/guide/read_and_write.md
index ec6cde5173a..c7feb69144b 100644
--- a/docs/src/guide/read_and_write.md
+++ b/docs/src/guide/read_and_write.md
@@ -456,3 +456,146 @@ affected files are no longer part of any ANN index if they were before. Because
 of this, it's recommended to rewrite files before re-building indices.
 
 <!-- TODO: remove this last comment once stable row ids are default. -->
+
+### Cleanup old versions
+
+Lance is an immutable format — every write creates a new version. The new version
+only writes the data that changed, so an insert writes the new rows and an update
+rewrites the affected columns for the affected rows. Even a delete creates a
+small deletion file. However, old versions still reference the previous data
+files, so those files are kept on disk until explicitly removed. Over time this
+means storage grows with each operation — inserts, updates, and deletes alike.
+
+Keeping old versions has important benefits: readers that opened an older version
+can continue reading it without interference from concurrent writers, providing
+snapshot isolation. Old versions also enable time travel queries, letting you
+read the dataset as it existed at any prior point in time.
+
+`cleanup_old_versions` deletes old version metadata and any data files that are
+no longer referenced by any version, reclaiming the accumulated storage.
+
+!!! warning
+
+    Once old versions are cleaned up, time travel queries to those versions are
+    no longer possible. Choose your retention window (`older_than`) accordingly —
+    any version removed by cleanup cannot be recovered.
+
+```python
+import lance
+
+dataset = lance.dataset("./my_dataset.lance")
+dataset.cleanup_old_versions()
+```
+
+By default, versions older than 7 days are removed. You can override this with
+the `older_than` parameter (a `timedelta`):
+
+```python
+from datetime import timedelta
+
+dataset.cleanup_old_versions(older_than=timedelta(days=1))
+```
+
+!!! note
+
+    Tagged versions are exempt from cleanup. See [Tags and Branches](tags_and_branches.md)
+    for details.
+
+By default, Lance only removes files that it can **verify** are no longer needed.
+A file is verified when Lance can see that it was referenced by an older version
+and is no longer referenced by any newer version. However, some orphaned files
+cannot be verified this way — for example, files left behind by aborted or failed
+commits that were never recorded in any version. These files are
+indistinguishable from files being written by an in-progress operation.
+
+Cleanup will never delete the current (active) version. This means passing
+`older_than=timedelta(0)` is safe and will delete all versions except the current
+one.
+
+The `delete_unverified` flag enables a more aggressive strategy that will also
+delete these unverified files:
+
+```python
+dataset.cleanup_old_versions(
+    older_than=timedelta(hours=2),
+    delete_unverified=True,
+)
+```
+
+!!! danger
+
+    Only use `delete_unverified=True` when you are confident that no other
+    concurrent operation has been in-progress for longer than the `older_than`
+    duration. Lance uses the file's age to decide whether an unverified file is
+    safe to remove, so any operation that is still running past the `older_than`
+    window risks having its files deleted out from under it.
+
+    In particular, combining `delete_unverified=True` with `older_than=timedelta(0)`
+    is **extremely dangerous** — if any other operation is in-progress at all,
+    its data files may be deleted, leading to dataset corruption.
+
+### Automatic cleanup
+
+Instead of calling `cleanup_old_versions` manually, you can configure Lance to
+clean up old versions automatically during writes. When auto cleanup is enabled,
+Lance will run cleanup every *N* commits (the **interval**), removing versions
+older than a specified duration.
+
+Auto cleanup can be enabled when creating a new dataset:
+
+```python
+import lance
+import pyarrow as pa
+from lance.dataset import AutoCleanupConfig
+
+table = pa.table({"id": range(100)})
+ds = lance.write_dataset(
+    table,
+    "./my_dataset.lance",
+    auto_cleanup_options=AutoCleanupConfig(
+        interval=20,             # run cleanup every 20 commits
+        older_than_seconds=3600, # remove versions older than 1 hour
+    ),
+)
+```
+
+Or enabled on an existing dataset:
+
+```python
+ds = lance.dataset("./my_dataset.lance")
+ds.optimize.enable_auto_cleanup(
+    AutoCleanupConfig(
+        interval=20,
+        older_than_seconds=3600,
+    )
+)
+```
+
+And disabled again:
+
+```python
+ds.optimize.disable_auto_cleanup()
+```
+
+Auto cleanup parameters can also be set directly via dataset config keys:
+
+```python
+ds.update_config({
+    "lance.auto_cleanup.interval": "20",
+    "lance.auto_cleanup.older_than": "3600s",
+})
+```
+
+!!! warning
+
+    Auto cleanup runs as part of the commit path. If your writer does not have
+    delete permissions, or you are doing high-frequency writes where the extra
+    latency matters, pass `skip_auto_cleanup=True` to `write_dataset` to skip it
+    on a per-write basis.
+
+### Other cleanup strategies
+
+It is common to run cleanup as a periodic background task on a dedicated server
+(for example, via a cron job or scheduled workflow). This keeps cleanup off the
+write path entirely, avoiding any impact to write latency, but requires setting
+up and maintaining additional infrastructure.
diff --git a/docs/src/images/mem_wal_overview.png b/docs/src/images/mem_wal_overview.png
index 008c84d0724..1c9f64bc647 100644
Binary files a/docs/src/images/mem_wal_overview.png and b/docs/src/images/mem_wal_overview.png differ
diff --git a/docs/src/images/mem_wal_regional.png b/docs/src/images/mem_wal_regional.png
deleted file mode 100644
index 5681fa27b8b..00000000000
Binary files a/docs/src/images/mem_wal_regional.png and /dev/null differ
diff --git a/docs/src/images/mem_wal_shard.png b/docs/src/images/mem_wal_shard.png
new file mode 100644
index 00000000000..ad8bed9e4b9
Binary files /dev/null and b/docs/src/images/mem_wal_shard.png differ
diff --git a/docs/src/integrations/.pages b/docs/src/integrations/.pages
index 62feffae067..ba764bf40f0 100644
--- a/docs/src/integrations/.pages
+++ b/docs/src/integrations/.pages
@@ -3,7 +3,7 @@ nav:
   - Apache DataFusion: datafusion.md
   - PostgreSQL: https://github.com/lancedb/pglance
   - PyTorch: pytorch.md
-  - Tensorflow: tensorflow.md
+  - TensorFlow: tensorflow.md
   - Apache Spark: spark
   - Ray: ray
   - Trino: trino
diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md
index 0304f8f5277..eb3b25051cc 100644
--- a/docs/src/integrations/index.md
+++ b/docs/src/integrations/index.md
@@ -27,7 +27,7 @@ GitHub organization.
 | Integration | Description | Source |
 |---|---|---|
 | [PyTorch](pytorch.md) | Use `lance.torch.data.LanceDataset` as a `torch.utils.data.IterableDataset` for training and inference. | Built-in |
-| [TensorFlow](tensorflow.md) | Use `lance.tf.data.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | Built-in |
+| [TensorFlow](tensorflow.md) | Use `lance_tensorflow.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | [lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) |
 | [Ray](ray) | Distributed read/write of Lance datasets with Ray Data. | [lance-format/lance-ray](https://github.com/lance-format/lance-ray) |
 | [Hugging Face](huggingface) | Convert and load Hugging Face datasets to and from Lance in a single call. | [lance-format/lance-huggingface](https://github.com/lance-format/lance-huggingface) |
 
diff --git a/docs/src/integrations/tensorflow.md b/docs/src/integrations/tensorflow.md
index 1c5d6b87157..03a5c5c5eca 100644
--- a/docs/src/integrations/tensorflow.md
+++ b/docs/src/integrations/tensorflow.md
@@ -1,92 +1,71 @@
-# Tensorflow Integration
+---
+title: TensorFlow
+description: Stream Lance datasets into TensorFlow tf.data pipelines with lance-tensorflow.
+---
 
-Lance can be used as a regular [tf.data.Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
-in [Tensorflow](https://www.tensorflow.org/).
+# TensorFlow Integration
 
-!!! warning
+The TensorFlow integration is maintained in the
+[lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow)
+project.
 
-    This feature is experimental and the APIs may change in the future.
+The main Lance Python package no longer includes `lance.tf`. Install
+`lance-tensorflow` and import `lance_tensorflow` instead.
+
+```bash
+pip install lance-tensorflow
+```
 
 ## Reading from Lance
 
-Using `lance.tf.data.from_lance`, you can create an `tf.data.Dataset` easily.
+Use `lance_tensorflow.from_lance` to create a `tf.data.Dataset` from a Lance
+dataset.
 
 ```python
-import tensorflow as tf
-import lance
+from lance_tensorflow import from_lance
 
-# Create tf dataset
-ds = lance.tf.data.from_lance("s3://my-bucket/my-dataset")
-
-# Chain tf dataset with other tf primitives
+ds = from_lance(
+    "s3://my-bucket/my-dataset",
+    columns=["image", "label"],
+    filter="split = 'train'",
+    batch_size=256,
+)
 
-for batch in ds.shuffling(32).map(lambda x: tf.io.decode_png(x["image"])):
-    print(batch)
+for batch in ds:
+    print(batch["label"])
 ```
 
-Backed by the Lance [columnar format](../format/index.md), using `lance.tf.data.from_lance` supports
-efficient column selection, filtering, and more.
+## Dataset Convenience Methods
+
+If you want `tf.data.Dataset.from_lance`, register the convenience methods
+explicitly after importing `lance_tensorflow`.
 
 ```python
-ds = lance.tf.data.from_lance(
-    "s3://my-bucket/my-dataset",
-    columns=["image", "label"],
-    filter="split = 'train' AND collected_time > timestamp '2020-01-01'",
-    batch_size=256)
-```
+import tensorflow as tf
+import lance_tensorflow
 
-By default, Lance will infer the Tensor spec from the projected columns. You can also specify `tf.TensorSpec` manually.
+lance_tensorflow.register_tensorflow_dataset()
 
-```python
-batch_size = 256
-ds = lance.tf.data.from_lance(
-    "s3://my-bucket/my-dataset",
-    columns=["image", "labels"],
-    batch_size=batch_size,
-    output_signature={
-        "image": tf.TensorSpec(shape=(), dtype=tf.string),
-        "labels": tf.RaggedTensorSpec(
-            dtype=tf.int32, shape=(batch_size, None), ragged_rank=1),
-    },
+ds = tf.data.Dataset.from_lance("s3://my-bucket/my-dataset")
 ```
 
-## Distributed Training and Shuffling
+## Migration
 
-Since [a Lance Dataset is a set of Fragments](../format/index.md), we can distribute and shuffle Fragments to different
-workers.
+Replace old imports:
 
 ```python
-import tensorflow as tf
-from lance.tf.data import from_lance, lance_fragments
+import lance.tf.data
 
-world_size = 32
-rank = 10
-seed = 123  #
-epoch = 100
+ds = lance.tf.data.from_lance(uri)
+```
 
-dataset_uri = "s3://my-bucket/my-dataset"
+with:
 
-# Shuffle fragments distributedly.
-fragments =
-    lance_fragments("s3://my-bucket/my-dataset")
-    .shuffling(32, seed=seed)
-    .repeat(epoch)
-    .enumerate()
-    .filter(lambda i, _: i % world_size == rank)
-    .map(lambda _, fid: fid)
+```python
+from lance_tensorflow import from_lance
 
-ds = from_lance(
-    uri,
-    columns=["image", "label"],
-    fragments=fragments,
-    batch_size=32
-    )
-for batch in ds:
-    print(batch)
+ds = from_lance(uri)
 ```
 
-!!! warning
-
-    For multiprocessing you should probably not use fork as lance is
-    multi-threaded internally and fork and multi-thread do not work well.
-    Refer to [this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555). 
\ No newline at end of file
+See the [lance-tensorflow README](https://github.com/lance-format/lance-tensorflow)
+for the current installation and compatibility details.
diff --git a/docs/src/quickstart/full-text-search.md b/docs/src/quickstart/full-text-search.md
index f990b2bd589..7f09c4325b7 100644
--- a/docs/src/quickstart/full-text-search.md
+++ b/docs/src/quickstart/full-text-search.md
@@ -98,6 +98,7 @@ ds.create_scalar_index(
     remove_stop_words=True,       # Remove stop words (language-dependent)
     custom_stop_words=None,       # Optional additional stop words (only used if remove_stop_words=True)
     ascii_folding=True,           # Fold accents to ASCII when possible (e.g., "é" -> "e")
+    block_size=128,               # Posting block size: 128 or 256; 256 is experimental
 )
 ```
 
diff --git a/docs/theme/404.html b/docs/theme/404.html
new file mode 100644
index 00000000000..51897bd0805
--- /dev/null
+++ b/docs/theme/404.html
@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+
+{% block container %}
+  <main class="ld-notfound">
+    <div class="ld-kicker">404 · Page not found</div>
+    <h1>This page doesn't exist.</h1>
+    <p>The page you're looking for may have moved.</p>
+    <div class="ld-hero__cta">
+      <a class="ld-btn ld-btn--primary ld-btn--lg" href="{{ 'index.html' | url }}">Back to home</a>
+      <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'quickstart/' | url }}">Getting started</a>
+    </div>
+  </main>
+{% endblock %}
diff --git a/docs/theme/assets/site.css b/docs/theme/assets/site.css
new file mode 100644
index 00000000000..a9d5286244c
--- /dev/null
+++ b/docs/theme/assets/site.css
@@ -0,0 +1,792 @@
+/* Lance docs site chrome — faithful port of the "Lance Docs" design prototype.
+   Layout/nav classes (ld-*) come from the theme templates; article content
+   styles target python-markdown + pymdownx output. */
+
+[hidden] { display: none !important; }
+
+.ld-shell {
+  min-height: 100vh;
+  display: flex;
+  flex-direction: column;
+  background: var(--surface-page);
+  font-family: var(--font-body);
+  color: var(--text-body);
+}
+
+/* ---------- header ---------- */
+.ld-header {
+  position: sticky;
+  top: 0;
+  z-index: 60;
+  display: flex;
+  align-items: center;
+  gap: 20px;
+  height: 60px;
+  padding: 0 clamp(16px, 3vw, 32px);
+  min-width: 0;
+  background: var(--surface-header);
+  backdrop-filter: blur(12px);
+  border-bottom: 1px solid var(--line-1);
+}
+.ld-brand {
+  display: inline-flex;
+  align-items: center;
+  text-decoration: none;
+  user-select: none;
+  flex-shrink: 0;
+}
+.ld-brand img { display: block; height: 24px; width: auto; }
+.ld-topnav {
+  display: flex;
+  gap: 2px;
+  margin-right: auto;
+  overflow-x: auto;
+  scrollbar-width: none;
+  min-width: 0;
+  flex: 1;
+}
+.ld-topnav::-webkit-scrollbar { display: none; }
+.ld-toptab {
+  font-family: var(--font-body);
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--text-muted);
+  text-decoration: none;
+  padding: 7px 12px;
+  cursor: pointer;
+  transition: color var(--dur-fast) var(--ease-out);
+  position: relative;
+  white-space: nowrap;
+}
+.ld-toptab:hover { color: var(--text-body); }
+.ld-toptab.active { color: var(--fg-1); }
+.ld-toptab.active::after {
+  content: "";
+  position: absolute;
+  left: 12px;
+  right: 12px;
+  bottom: -10px;
+  height: 2px;
+  background: var(--beam-400);
+}
+.ld-header__actions { display: flex; gap: 10px; align-items: center; flex-shrink: 0; }
+/* All header controls share one 34px height regardless of content (icon/text). */
+.ld-header__actions .ld-btn { height: 34px; padding-top: 0; padding-bottom: 0; }
+.ld-header__actions .ld-btn--icon { width: 34px; padding: 0; justify-content: center; }
+
+/* ---------- buttons ---------- */
+.ld-btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 8px;
+  text-decoration: none;
+  white-space: nowrap;
+  flex-shrink: 0;
+}
+.ld-btn--primary {
+  font-family: var(--font-body);
+  font-size: 13.5px;
+  font-weight: 500;
+  color: #ffffff;
+  background: var(--beam-400);
+  padding: 8px 16px;
+}
+.ld-btn--primary:hover { background: var(--beam-600); color: #ffffff; }
+.ld-btn--ghost {
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  color: var(--text-secondary);
+  border: 1px solid var(--line-1);
+  padding: 7px 14px;
+}
+.ld-btn--ghost:hover { border-color: var(--beam-400); color: var(--fg-1); }
+.ld-btn--icon { padding: 7px 9px; background: none; cursor: pointer; }
+.ld-btn__count {
+  border-left: 1px solid var(--line-2);
+  padding-left: 8px;
+  color: var(--fg-1);
+  font-weight: 600;
+}
+:root[data-theme="light"] .ld-icon-sun { display: none; }
+:root[data-theme="dark"] .ld-icon-moon { display: none; }
+.ld-btn--outline {
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--fg-1);
+  border: 1px solid var(--line-1);
+  padding: 11px 24px;
+}
+.ld-btn--outline:hover { border-color: var(--beam-400); color: var(--fg-1); }
+.ld-btn--text {
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--text-secondary);
+  padding: 11px 12px;
+}
+.ld-btn--text:hover { color: var(--fg-1); }
+.ld-btn--lg.ld-btn--primary { font-size: 14px; padding: 12px 24px; }
+
+/* ---------- home: hero ---------- */
+.ld-hero { max-width: var(--container-max); margin: 0 auto; padding: 96px 32px 72px; width: 100%; }
+.ld-kicker {
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--beam-600);
+  margin-bottom: 24px;
+}
+/* Wrap only between segments (at the · separators), never inside one. */
+.ld-kicker span, .ld-kicker a { white-space: nowrap; }
+.ld-kicker a { color: inherit; text-decoration: underline; text-underline-offset: 3px; }
+.ld-kicker a:hover { color: var(--beam-700); }
+.ld-hero h1 {
+  font-family: var(--font-display);
+  font-size: 72px;
+  font-weight: 600;
+  letter-spacing: var(--tracking-display);
+  line-height: 0.97;
+  color: var(--fg-1);
+  margin: 0 0 28px;
+  max-width: 900px;
+  text-wrap: balance;
+}
+.ld-hero__lead {
+  font-size: 17px;
+  line-height: 1.6;
+  color: var(--text-secondary);
+  max-width: 640px;
+  margin: 0 0 36px;
+  text-wrap: pretty;
+}
+.ld-hero__cta { display: flex; gap: 12px; flex-wrap: wrap; }
+
+/* ---------- home: stats band ---------- */
+.ld-band { max-width: var(--container-max); margin: 0 auto; padding: 0 32px; width: 100%; }
+.ld-stats {
+  display: grid;
+  grid-template-columns: 1fr 1fr 1fr;
+  border-top: 1px solid var(--line-1);
+  border-bottom: 1px solid var(--line-1);
+}
+.ld-stat { padding: 28px 32px; }
+.ld-stat:first-child { padding-left: 0; }
+.ld-stat:last-child { padding-right: 0; }
+.ld-stat + .ld-stat { border-left: 1px solid var(--line-2); }
+.ld-stat--link { display: block; text-decoration: none; }
+.ld-stat--link .ld-stat__label { transition: color var(--dur-fast) var(--ease-out); }
+.ld-stat--link:hover .ld-stat__label { color: var(--beam-600); }
+.ld-stat__value {
+  font-family: var(--font-display);
+  font-size: 40px;
+  font-weight: 600;
+  letter-spacing: -0.03em;
+  color: var(--fg-1);
+}
+.ld-stat__value--beam { color: var(--beam-400); }
+.ld-stat__label {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  letter-spacing: 0.1em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin-top: 6px;
+}
+
+/* ---------- home: what is lance ---------- */
+.ld-what { max-width: var(--container-max); margin: 0 auto; padding: 72px 32px 40px; width: 100%; }
+.ld-what h2 {
+  font-family: var(--font-display);
+  font-size: 34px;
+  font-weight: 600;
+  letter-spacing: -0.03em;
+  color: var(--fg-1);
+  margin: 0 0 16px;
+}
+.ld-what__body {
+  font-size: 16px;
+  line-height: 1.65;
+  color: var(--text-secondary);
+  max-width: 760px;
+  margin: 0 0 12px;
+  text-wrap: pretty;
+}
+.ld-what__more { font-size: 15px; line-height: 1.6; color: var(--text-secondary); margin: 0; }
+.ld-inline-link {
+  color: var(--beam-600);
+  text-decoration: underline;
+  text-decoration-thickness: 2px;
+  text-underline-offset: 3px;
+}
+
+/* ---------- home: features ---------- */
+.ld-features { max-width: var(--container-max); margin: 0 auto; padding: 24px 32px 96px; width: 100%; }
+.ld-feature {
+  display: grid;
+  grid-template-columns: 90px minmax(0, 1fr) minmax(0, 460px);
+  gap: 40px;
+  align-items: center;
+  border-top: 1px solid var(--line-1);
+  padding: 40px 0;
+}
+.ld-feature__num {
+  font-family: var(--font-mono);
+  font-size: 15px;
+  color: var(--beam-600);
+  align-self: start;
+  padding-top: 6px;
+}
+.ld-feature__body h3 {
+  font-family: var(--font-display);
+  font-size: 24px;
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  color: var(--fg-1);
+  margin: 0 0 12px;
+}
+.ld-feature__body p {
+  font-size: 15px;
+  line-height: 1.65;
+  color: var(--text-secondary);
+  margin: 0 0 16px;
+  text-wrap: pretty;
+}
+.ld-more {
+  font-family: var(--font-mono);
+  font-size: 13px;
+  color: var(--beam-600);
+  text-decoration: none;
+}
+.ld-more:hover { text-decoration: underline; }
+/* Logo grids stay on white in both themes — the artwork assumes a light background. */
+.ld-feature__img { border: 1px solid var(--line-1); padding: 16px; background: #ffffff; }
+.ld-feature__img img { display: block; width: 100%; height: auto; }
+
+/* homepage code windows (always ink-dark, like docs code blocks) */
+.ld-feature__code { border: 1px solid var(--line-2); background: var(--ink-900); min-width: 0; border-radius: var(--radius-box); overflow: hidden; }
+.ld-win { display: flex; gap: 6px; padding: 12px 16px 0; }
+.ld-win i { width: 10px; height: 10px; border-radius: 999px; background: var(--ink-600); }
+.ld-feature__code pre {
+  margin: 0;
+  padding: 14px 20px 18px;
+  overflow-x: auto;
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  line-height: 1.7;
+  color: #EDEBF5;
+}
+.ld-feature__code code { font-family: inherit; }
+.tok-kw { color: #B8A8FF; }
+.tok-str { color: #7CC0FF; }
+.tok-com { color: #6F6A85; font-style: italic; }
+.tok-num { color: #F2CE6B; }
+.tok-arg { color: #FF95A8; }
+.tok-fn { color: #8FE8CE; }
+
+/* ---------- docs layout ---------- */
+.ld-docs {
+  flex: 1;
+  display: grid;
+  grid-template-columns: 240px minmax(0, 1fr) 190px;
+  max-width: 1280px;
+  width: 100%;
+  margin: 0 auto;
+  gap: 44px;
+  padding: 0 32px;
+}
+.ld-sidenav {
+  border-right: 1px solid var(--line-2);
+  padding: 36px 24px 48px 0;
+  position: sticky;
+  top: 60px;
+  align-self: start;
+  height: calc(100vh - 60px);
+  overflow-y: auto;
+}
+/* Mobile-only disclosure for the sidenav; hidden on desktop where the
+   sidenav is a sticky column. */
+.ld-sidenav-toggle {
+  display: none;
+  align-items: center;
+  justify-content: space-between;
+  gap: 8px;
+  width: 100%;
+  font-family: var(--font-mono);
+  font-size: 11px;
+  font-weight: 600;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-secondary);
+  background: none;
+  border: 1px solid var(--line-2);
+  padding: 9px 12px;
+  margin: 16px 0 0;
+  cursor: pointer;
+}
+.ld-sidenav-toggle:hover { color: var(--fg-1); border-color: var(--beam-400); }
+.ld-sidenav-toggle svg { transition: transform var(--dur-fast) var(--ease-out); }
+.ld-sidenav-toggle.open svg { transform: rotate(180deg); }
+.ld-sidenav__group { margin-bottom: 6px; }
+.ld-sidenav__label {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 22px 0 8px;
+}
+.ld-sidenav a {
+  display: block;
+  font-size: 13.5px;
+  color: var(--text-secondary);
+  text-decoration: none;
+  padding: 5px 12px;
+  cursor: pointer;
+  border-left: 1px solid var(--line-2);
+  line-height: 1.45;
+}
+.ld-sidenav a:hover { color: var(--text-body); background: var(--surface-overlay); }
+.ld-sidenav a.active {
+  color: var(--beam-600);
+  border-left: 2px solid var(--beam-400);
+  padding-left: 11px;
+  font-weight: 500;
+}
+.ld-sidenav a .ext { color: var(--text-muted); font-size: 11px; margin-left: 4px; }
+
+/* ---------- right-hand page TOC ---------- */
+.ld-toc {
+  position: sticky;
+  top: 60px;
+  align-self: start;
+  max-height: calc(100vh - 60px);
+  overflow-y: auto;
+  padding: 44px 0 48px;
+}
+.ld-toc__label {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 0 0 10px;
+}
+.ld-toc a {
+  display: block;
+  font-size: 12.5px;
+  line-height: 1.45;
+  color: var(--text-muted);
+  text-decoration: none;
+  border-left: 1px solid var(--line-2);
+  padding: 4px 0 4px 12px;
+}
+.ld-toc a.ld-toc__h3 { padding-left: 24px; }
+.ld-toc a:hover { color: var(--text-body); }
+.ld-toc a.active {
+  color: var(--beam-600);
+  border-left: 2px solid var(--beam-400);
+  padding-left: 11px;
+  font-weight: 500;
+}
+.ld-toc a.ld-toc__h3.active { padding-left: 23px; }
+
+.ld-crumbs {
+  font-family: var(--font-mono);
+  font-size: 11.5px;
+  color: var(--text-muted);
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+  margin-bottom: 24px;
+}
+.ld-crumbs b { color: var(--beam-600); font-weight: 500; }
+
+.ld-pagenav {
+  display: flex;
+  justify-content: space-between;
+  gap: 16px;
+  border-top: 1px solid var(--line-1);
+  margin-top: 56px;
+  padding-top: 20px;
+}
+.ld-pagenav a { text-decoration: none; color: var(--fg-1); }
+.ld-pagenav a:hover { color: var(--beam-600); }
+.ld-pagenav a.next { text-align: right; }
+.ld-pagenav__k {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin-bottom: 4px;
+}
+.ld-pagenav__t { font-size: 14.5px; font-weight: 500; }
+
+/* ---------- article prose (python-markdown output) ---------- */
+.ld-article-col { max-width: 760px; min-width: 0; padding: 40px 0 96px; }
+.ld-article { font-family: var(--font-body); color: var(--text-secondary); }
+/* Keep anchor targets clear of the sticky header. */
+.ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 76px; }
+.ld-article h1 { font-family: var(--font-display); font-size: 42px; font-weight: 600; letter-spacing: -0.03em; line-height: 1.05; color: var(--fg-1); margin: 0 0 20px; text-wrap: balance; }
+.ld-article h2 { font-family: var(--font-display); font-size: 26px; font-weight: 600; letter-spacing: -0.02em; color: var(--fg-1); margin: 48px 0 14px; padding-top: 28px; border-top: 1px solid var(--line-2); }
+.ld-article h3 { font-family: var(--font-display); font-size: 19px; font-weight: 600; letter-spacing: -0.01em; color: var(--fg-1); margin: 32px 0 10px; }
+.ld-article h4, .ld-article h5, .ld-article h6 { font-family: var(--font-mono); font-size: 12.5px; font-weight: 600; letter-spacing: .14em; text-transform: uppercase; color: var(--fg-1); margin: 28px 0 8px; }
+.ld-article p { font-size: 15.5px; line-height: 1.65; margin: 0 0 16px; text-wrap: pretty; }
+.ld-article a { color: var(--beam-600); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 3px; text-decoration-color: rgba(98, 94, 255, .35); }
+.ld-article a:hover { text-decoration-color: var(--beam-600); }
+.ld-article code { font-family: var(--font-mono); font-size: 13px; background: var(--surface-inline-code); border-radius: var(--radius-chip); padding: 1px 5px; color: var(--fg-1); }
+.ld-article ul, .ld-article ol { margin: 0 0 16px; padding-left: 22px; font-size: 15.5px; line-height: 1.65; }
+.ld-article li { margin-bottom: 6px; }
+.ld-article li p { margin-bottom: 8px; }
+.ld-article hr { border: 0; border-top: 1px solid var(--line-1); margin: 40px 0; }
+.ld-article blockquote { margin: 0 0 16px; padding: 4px 0 4px 20px; border-left: 2px solid var(--beam-400); }
+.ld-article blockquote p:last-child { margin-bottom: 0; }
+
+/* heading permalinks (toc: permalink) */
+.ld-article .headerlink {
+  margin-left: 8px;
+  color: var(--beam-400);
+  text-decoration: none;
+  opacity: 0;
+  transition: opacity var(--dur-fast) var(--ease-out);
+}
+.ld-article :is(h1, h2, h3, h4, h5, h6):hover .headerlink { opacity: 1; }
+
+/* figures: lone images sit on a white plate in both themes */
+.ld-article img { max-width: 100%; height: auto; }
+.ld-article p > img:only-child {
+  display: block;
+  margin: 24px auto;
+  border: 1px solid var(--line-1);
+  padding: 16px;
+  background: #ffffff;
+}
+
+/* tables (JS wraps them in .ld-tablewrap for overflow) */
+.ld-tablewrap { overflow-x: auto; margin: 0 0 20px; border: 1px solid var(--line-1); }
+.ld-article table { border-collapse: collapse; width: 100%; font-size: 14px; }
+.ld-article th { font-family: var(--font-mono); font-size: 11px; font-weight: 600; letter-spacing: .12em; text-transform: uppercase; text-align: left; color: var(--fg-1); background: var(--surface-overlay); padding: 10px 14px; border-bottom: 1px solid var(--line-1); }
+.ld-article td { padding: 10px 14px; border-bottom: 1px solid var(--line-2); vertical-align: top; line-height: 1.55; color: var(--text-secondary); }
+.ld-article tr:last-child td { border-bottom: 0; }
+
+/* ---------- code blocks (pymdownx.highlight output, JS adds the bar) ---------- */
+.ld-code { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; }
+.ld-code__bar { display: flex; justify-content: space-between; align-items: center; padding: 7px 14px; border-bottom: 1px solid var(--line-2); background: var(--surface-card); }
+.ld-code__bar span { display: inline-flex; align-items: center; gap: 7px; font-family: var(--font-mono); font-size: 11px; letter-spacing: .12em; text-transform: uppercase; color: var(--text-muted); }
+.ld-code__bar span svg { width: 13px; height: 13px; display: block; flex-shrink: 0; }
+.ld-copy { font-family: var(--font-mono); font-size: 11px; letter-spacing: .08em; text-transform: uppercase; background: none; border: none; color: var(--beam-600); cursor: pointer; padding: 2px 0; }
+.ld-copy:hover { color: var(--beam-700); }
+.ld-code .highlight { margin: 0; }
+.ld-code pre { margin: 0; background: var(--surface-code); padding: 16px 18px; overflow-x: auto; }
+.ld-code pre code { background: none; border: none; padding: 0; font-size: 13px; line-height: 1.6; color: var(--code-fg); }
+/* Ink code surfaces (dark-theme docs blocks + the always-dark homepage windows)
+   need an explicit light selection color; light-theme docs blocks use the default. */
+:root[data-theme="dark"] .ld-code pre ::selection, :root[data-theme="dark"] .ld-code pre::selection,
+.ld-feature__code pre ::selection, .ld-feature__code pre::selection {
+  background: rgba(98, 94, 255, 0.55);
+  color: #ffffff;
+}
+
+/* Pygments tokens — colors resolve per theme via the --code-* palette */
+.highlight .k, .highlight .kn, .highlight .kd, .highlight .kt, .highlight .kr, .highlight .kp, .highlight .ow { color: var(--code-kw); }
+.highlight .kc { color: var(--code-kw); }
+.highlight .s, .highlight .s1, .highlight .s2, .highlight .sb, .highlight .sd, .highlight .sa, .highlight .se, .highlight .si, .highlight .sx, .highlight .sr, .highlight .ss, .highlight .sh { color: var(--code-str); }
+.highlight .c, .highlight .c1, .highlight .cm, .highlight .ch, .highlight .cs, .highlight .cp, .highlight .cpf { color: var(--code-com); font-style: italic; }
+.highlight .m, .highlight .mi, .highlight .mf, .highlight .mh, .highlight .mo, .highlight .mb, .highlight .il { color: var(--code-num); }
+.highlight .nf, .highlight .fm, .highlight .nd, .highlight .ne { color: var(--code-fn); }
+.highlight .nc, .highlight .nn { color: var(--code-name); font-weight: 500; }
+.highlight .nb, .highlight .bp { color: var(--code-kw); }
+.highlight .nt { color: var(--code-tag); }
+.highlight .na { color: var(--code-tag); }
+.highlight .o { color: var(--code-punct); }
+.highlight .p { color: var(--code-punct); }
+.highlight .gp { color: var(--code-prompt); }
+.highlight .go { color: var(--code-punct); }
+.highlight .gh, .highlight .gu { color: var(--code-name); font-weight: 600; }
+.highlight .hll { background: var(--code-hll); display: block; }
+
+/* mermaid diagrams: white plate (rendered by JS) */
+.ld-article pre.mermaid, .ld-article div.mermaid {
+  border: 1px solid var(--line-1);
+  padding: 16px;
+  background: #ffffff;
+  margin: 0 0 20px;
+  text-align: center;
+  overflow-x: auto;
+}
+
+/* ---------- admonitions ---------- */
+.ld-article .admonition {
+  border: 1px solid var(--line-2);
+  border-radius: var(--radius-box);
+  overflow: hidden;
+  margin: 0 0 20px;
+  background: var(--surface-card);
+  padding: 0;
+  font-size: 14.5px;
+}
+.ld-article .admonition > .admonition-title {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-family: var(--font-mono);
+  font-size: 11px;
+  font-weight: 600;
+  letter-spacing: .14em;
+  text-transform: uppercase;
+  color: var(--beam-600);
+  padding: 9px 16px;
+  margin: 0;
+  border-bottom: 1px solid var(--line-2);
+  background: none;
+}
+/* Material-style type icon, drawn in the title color via a CSS mask */
+.ld-article .admonition > .admonition-title::before {
+  content: "";
+  width: 14px;
+  height: 14px;
+  flex-shrink: 0;
+  background-color: currentColor;
+  -webkit-mask: var(--ld-adm-icon) no-repeat center / contain;
+  mask: var(--ld-adm-icon) no-repeat center / contain;
+}
+/* Icon shapes are Material Design Icons paths (Apache 2.0), matching mkdocs-material's defaults. */
+.ld-article .admonition { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.abstract, .admonition.summary, .admonition.tldr) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.info, .admonition.todo) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,9H11V7H13M13,17H11V11H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.tip, .admonition.hint, .admonition.important) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17.66,11.2C17.43,10.9 17.15,10.64 16.89,10.38C16.22,9.78 15.46,9.35 14.82,8.72C13.33,7.26 13,4.85 13.95,3C13,3.23 12.17,3.75 11.46,4.32C8.87,6.4 7.85,10.07 9.07,13.22C9.11,13.32 9.15,13.42 9.15,13.55C9.15,13.77 9,13.97 8.8,14.05C8.57,14.15 8.33,14.09 8.14,13.93C8.08,13.88 8.04,13.83 8,13.76C6.87,12.33 6.69,10.28 7.45,8.64C5.78,10 4.87,12.3 5,14.47C5.06,14.97 5.12,15.47 5.29,15.97C5.43,16.57 5.7,17.17 6,17.7C7.08,19.43 8.95,20.67 10.96,20.92C13.1,21.19 15.39,20.8 17.03,19.32C18.86,17.66 19.5,15 18.56,12.72L18.43,12.46C18.22,12 17.66,11.2 17.66,11.2M14.5,17.5C14.22,17.74 13.76,18 13.4,18.1C12.28,18.5 11.16,17.94 10.5,17.28C11.69,17 12.4,16.12 12.61,15.23C12.78,14.43 12.46,13.77 12.33,13C12.21,12.26 12.23,11.63 12.5,10.94C12.69,11.32 12.89,11.7 13.13,12C13.9,13 15.11,13.44 15.37,14.8C15.41,14.94 15.43,15.08 15.43,15.23C15.46,16.05 15.1,16.95 14.5,17.5H14.5Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.success, .admonition.check, .admonition.done) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M9,20.42L2.79,14.21L5.62,11.38L9,14.77L18.88,4.88L21.71,7.71L9,20.42Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.question, .admonition.help, .admonition.faq) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M15.07,11.25L14.17,12.17C13.45,12.89 13,13.5 13,15H11V14.5C11,13.39 11.45,12.39 12.17,11.67L13.41,10.41C13.78,10.05 14,9.55 14,9C14,7.89 13.1,7 12,7A2,2 0 0,0 10,9H8A4,4 0 0,1 12,5A4,4 0 0,1 16,9C16,9.88 15.64,10.67 15.07,11.25M13,19H11V17H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12C22,6.47 17.5,2 12,2Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.warning, .admonition.caution, .admonition.attention) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,14H11V9H13M13,18H11V16H13M1,21H23L12,2L1,21Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.failure, .admonition.fail, .admonition.missing) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20 6.91L17.09 4L12 9.09L6.91 4L4 6.91L9.09 12L4 17.09L6.91 20L12 14.91L17.09 20L20 17.09L14.91 12L20 6.91Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.danger, .admonition.error) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M11,15H6L13,1V9H18L11,23V15Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition.bug { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,12H10V10H14M14,16H10V14H14M20,8H17.19C16.74,7.22 16.12,6.55 15.37,6.04L17,4.41L15.59,3L13.42,5.17C12.96,5.06 12.5,5 12,5C11.5,5 11.04,5.06 10.59,5.17L8.41,3L7,4.41L8.62,6.04C7.88,6.55 7.26,7.22 6.81,8H4V10H6.09C6.04,10.33 6,10.66 6,11V12H4V14H6V15C6,15.34 6.04,15.67 6.09,16H4V18H6.81C7.85,19.79 9.78,21 12,21C14.22,21 16.15,19.79 17.19,18H20V16H17.91C17.96,15.67 18,15.34 18,15V14H20V12H18V11C18,10.66 17.96,10.33 17.91,10H20V8Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition.example { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M7,13V11H21V13H7M7,19V17H21V19H7M7,7V5H21V7H7M3,8V5H2V4H4V8H3M2,17V16H5V20H2V19H4V18.5H3V17.5H4V17H2M4.25,10A0.75,0.75 0 0,1 5,10.75C5,10.95 4.92,11.14 4.79,11.27L3.12,13H5V14H2V13.08L4,11H2V10H4.25Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.quote, .admonition.cite) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,17H17L19,13V7H13V13H16M6,17H9L11,13V7H5V13H8L6,17Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition > :not(.admonition-title) { margin: 14px 16px; font-size: 14.5px; }
+.ld-article .admonition > .highlight, .ld-article .admonition > .ld-code { margin: 14px 16px; }
+.ld-article .admonition.warning > .admonition-title,
+.ld-article .admonition.caution > .admonition-title,
+.ld-article .admonition.attention > .admonition-title { color: var(--warn-500); }
+.ld-article .admonition.danger > .admonition-title,
+.ld-article .admonition.error > .admonition-title,
+.ld-article .admonition.bug > .admonition-title,
+.ld-article .admonition.failure > .admonition-title { color: var(--danger-500); }
+.ld-article .admonition.success > .admonition-title,
+.ld-article .admonition.check > .admonition-title { color: var(--ok-500); }
+
+/* ---------- details / summary (pymdownx.details) ---------- */
+.ld-article details {
+  border: 1px solid var(--line-2);
+  border-radius: var(--radius-box);
+  overflow: hidden;
+  margin: 0 0 20px;
+  background: var(--surface-card);
+}
+.ld-article details > summary {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  font-weight: 600;
+  letter-spacing: .1em;
+  text-transform: uppercase;
+  color: var(--beam-600);
+  padding: 10px 16px;
+  cursor: pointer;
+  user-select: none;
+  list-style: none;
+}
+.ld-article details > summary::-webkit-details-marker { display: none; }
+.ld-article details > summary:hover { background: var(--surface-overlay); }
+.ld-article details[open] > summary { border-bottom: 1px solid var(--line-2); }
+.ld-article details > :not(summary) { margin: 14px 16px; }
+
+/* ---------- content tabs (pymdownx.tabbed, alternate style) ---------- */
+.ld-article .tabbed-set { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; position: relative; }
+.ld-article .tabbed-set > input { position: absolute; opacity: 0; pointer-events: none; }
+.ld-article .tabbed-labels {
+  display: flex;
+  border-bottom: 1px solid var(--line-2);
+  background: var(--surface-card);
+  overflow-x: auto;
+  scrollbar-width: none;
+}
+.ld-article .tabbed-labels > label {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  letter-spacing: .06em;
+  padding: 9px 18px;
+  border-right: 1px solid var(--line-2);
+  color: var(--text-muted);
+  cursor: pointer;
+  white-space: nowrap;
+}
+.ld-article .tabbed-labels > label:hover { color: var(--fg-1); }
+.ld-article .tabbed-content { display: block; }
+.ld-article .tabbed-block { display: none; padding: 16px 16px 0; }
+.ld-article .tabbed-block > :last-child { margin-bottom: 16px; }
+.ld-article .tabbed-block > .ld-code:last-child { margin-bottom: 16px; }
+/* nth-input → nth-label/nth-block mapping (supports up to 8 tabs) */
+.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-labels > label:nth-child(1),
+.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > label:nth-child(2),
+.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > label:nth-child(3),
+.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > label:nth-child(4),
+.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > label:nth-child(5),
+.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > label:nth-child(6),
+.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > label:nth-child(7),
+.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > label:nth-child(8) {
+  color: var(--fg-1);
+  box-shadow: inset 0 -2px 0 var(--beam-400);
+}
+.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-content > .tabbed-block:nth-child(1),
+.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-content > .tabbed-block:nth-child(2),
+.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-content > .tabbed-block:nth-child(3),
+.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-content > .tabbed-block:nth-child(4),
+.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-content > .tabbed-block:nth-child(5),
+.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-content > .tabbed-block:nth-child(6),
+.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-content > .tabbed-block:nth-child(7),
+.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-content > .tabbed-block:nth-child(8) {
+  display: block;
+}
+
+/* ---------- search overlay ---------- */
+.ld-search { position: fixed; inset: 0; z-index: 100; }
+.ld-search__scrim { position: absolute; inset: 0; background: rgba(14, 12, 20, 0.5); }
+.ld-search__panel {
+  position: relative;
+  max-width: 640px;
+  margin: 96px auto 0;
+  background: var(--surface-page);
+  border: 1px solid var(--line-1);
+}
+.ld-search__bar { display: flex; align-items: center; border-bottom: 1px solid var(--line-1); }
+.ld-search__input {
+  flex: 1;
+  font-family: var(--font-body);
+  font-size: 15px;
+  color: var(--fg-1);
+  background: none;
+  border: none;
+  outline: none;
+  padding: 14px 16px;
+}
+.ld-search__input::placeholder { color: var(--text-muted); }
+.ld-search__close {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: .1em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+  background: none;
+  border: 1px solid var(--line-2);
+  margin-right: 12px;
+  padding: 3px 8px;
+  cursor: pointer;
+}
+.ld-search__close:hover { color: var(--fg-1); border-color: var(--beam-400); }
+.ld-search__results { max-height: 55vh; overflow-y: auto; }
+.ld-search__hit { display: block; padding: 12px 16px; text-decoration: none; border-top: 1px solid var(--line-2); }
+.ld-search__hit:first-child { border-top: 0; }
+.ld-search__hit:hover, .ld-search__hit.active { background: var(--surface-overlay); }
+.ld-search__hit-title { font-size: 14px; font-weight: 500; color: var(--fg-1); }
+.ld-search__hit-title mark, .ld-search__hit-text mark { background: none; color: var(--beam-600); }
+.ld-search__hit-crumb { font-family: var(--font-mono); font-size: 10.5px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); margin-bottom: 2px; }
+.ld-search__hit-text { font-size: 12.5px; color: var(--text-muted); line-height: 1.5; margin-top: 2px; }
+.ld-search__empty { font-family: var(--font-mono); font-size: 12px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); padding: 20px 16px; }
+
+/* ---------- 404 ---------- */
+.ld-notfound { max-width: var(--container-max); margin: 0 auto; padding: 120px 32px 160px; width: 100%; flex: 1; }
+.ld-notfound h1 {
+  font-family: var(--font-display);
+  font-size: 56px;
+  font-weight: 600;
+  letter-spacing: var(--tracking-display);
+  color: var(--fg-1);
+  margin: 0 0 16px;
+}
+.ld-notfound p { color: var(--text-secondary); margin: 0 0 32px; }
+
+/* ---------- footer ---------- */
+.ld-footer {
+  border-top: 1px solid var(--line-1);
+  padding: 40px 32px;
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 32px;
+  flex-wrap: wrap;
+}
+.ld-footer__brand img { display: block; height: 24px; width: auto; }
+.ld-footer__brand p {
+  font-size: 12.5px;
+  color: var(--text-muted);
+  max-width: 300px;
+  line-height: 1.6;
+  margin: 12px 0 0;
+}
+.ld-footer__cols { display: flex; gap: 64px; flex-wrap: wrap; }
+.ld-footer__col { display: flex; flex-direction: column; gap: 8px; }
+.ld-footer__col h5 {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 0 0 4px;
+}
+.ld-footer__col a { font-size: 13.5px; color: var(--text-secondary); text-decoration: none; }
+.ld-footer__col a:hover { color: var(--fg-1); }
+
+/* ---------- responsive ---------- */
+@media (max-width: 1180px) {
+  .ld-docs { grid-template-columns: 240px minmax(0, 1fr); }
+  .ld-toc { display: none; }
+}
+@media (max-width: 960px) {
+  /* Header folds into two rows: brand + icon actions, then a scrollable
+     section-tab row. Text-heavy controls collapse to icons. */
+  .ld-header { flex-wrap: wrap; height: auto; gap: 0 10px; padding: 10px 16px 0; }
+  .ld-header__actions { margin-left: auto; }
+  .ld-header__actions .ld-btn--primary,
+  .ld-header__actions .ld-btn__label,
+  .ld-header__actions .ld-btn__count { display: none; }
+  .ld-topnav { order: 1; flex-basis: 100%; margin: 4px 0 0; padding-bottom: 10px; }
+  .ld-toptab { padding: 7px 10px; }
+  .ld-toptab:first-child { margin-left: -10px; }
+  .ld-toptab.active::after { left: 10px; right: 10px; }
+
+  .ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 104px; }
+  .ld-article h1 { font-size: 32px; }
+
+  .ld-hero h1 { font-size: clamp(38px, 9vw, 56px); }
+  .ld-stats { grid-template-columns: 1fr; }
+  .ld-stat { padding: 20px 0; }
+  .ld-stat + .ld-stat { border-left: 0; border-top: 1px solid var(--line-2); }
+  .ld-feature { grid-template-columns: 1fr; gap: 16px; }
+  .ld-feature__num { padding-top: 0; }
+
+  /* Docs: sidenav collapses behind a disclosure button; article leads. */
+  .ld-docs { display: block; padding: 0 20px; }
+  .ld-sidenav-toggle { display: flex; }
+  .ld-sidenav {
+    display: none;
+    position: static;
+    height: auto;
+    border-right: 0;
+    border-bottom: 1px solid var(--line-2);
+    padding: 0 0 20px;
+  }
+  .ld-sidenav.open { display: block; }
+  .ld-sidenav__group:first-child .ld-sidenav__label { margin-top: 16px; }
+  .ld-article-col { padding-top: 24px; }
+
+  .ld-hero, .ld-band, .ld-what, .ld-features { padding-left: 20px; padding-right: 20px; }
+  .ld-hero { padding-top: 56px; padding-bottom: 48px; }
+  .ld-notfound { padding: 72px 20px 96px; }
+  .ld-footer { padding: 32px 20px; }
+  .ld-footer__cols { gap: 28px 40px; }
+}
+@media (max-width: 700px) {
+  /* Search becomes an edge-to-edge sheet under the top of the screen. */
+  .ld-search__panel { margin: 0; max-width: none; border-left: 0; border-right: 0; border-top: 0; }
+  .ld-search__results { max-height: calc(100dvh - 110px); }
+}
diff --git a/docs/theme/assets/site.js b/docs/theme/assets/site.js
new file mode 100644
index 00000000000..34378c820b2
--- /dev/null
+++ b/docs/theme/assets/site.js
@@ -0,0 +1,263 @@
+/* Lance docs theme behaviours: theme toggle, GitHub stars, code block chrome,
+   TOC scroll-spy, search overlay, mermaid rendering. */
+(function () {
+  "use strict";
+
+  var BASE = (window.LD_BASE || ".").replace(/\/$/, "");
+
+  /* ---------- theme toggle ---------- */
+  var themeBtn = document.getElementById("theme-toggle");
+  if (themeBtn) {
+    themeBtn.addEventListener("click", function () {
+      var next = document.documentElement.dataset.theme === "dark" ? "light" : "dark";
+      document.documentElement.dataset.theme = next;
+      try { localStorage.setItem("ld-theme", next); } catch (e) { /* private mode */ }
+    });
+  }
+
+  /* ---------- GitHub stars ---------- */
+  (function loadStars() {
+    var el = document.getElementById("gh-stars");
+    if (!el) return;
+    var TTL = 3600e3;
+    function show(count) {
+      if (!(count > 0)) return;
+      var label = count >= 1000 ? (count / 1000).toFixed(1).replace(/\.0$/, "") + "k" : String(count);
+      el.textContent = "★ " + label;
+      el.hidden = false;
+    }
+    try {
+      var cached = JSON.parse(localStorage.getItem("ld-gh-stars") || "null");
+      if (cached && Date.now() - cached.t < TTL) { show(cached.v); return; }
+    } catch (e) { /* fall through to fetch */ }
+    fetch("https://api.github.com/repos/lance-format/lance")
+      .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+      .then(function (d) {
+        show(d.stargazers_count);
+        try { localStorage.setItem("ld-gh-stars", JSON.stringify({ v: d.stargazers_count, t: Date.now() })); } catch (e) { /* ignore */ }
+      })
+      .catch(function () { /* rate-limited or offline: button still works without the count */ });
+  })();
+
+  /* ---------- mobile sidenav toggle ---------- */
+  var sidenavToggle = document.querySelector(".ld-sidenav-toggle");
+  var sidenav = document.querySelector(".ld-sidenav");
+  if (sidenavToggle && sidenav) {
+    sidenavToggle.addEventListener("click", function () {
+      var open = sidenav.classList.toggle("open");
+      sidenavToggle.classList.toggle("open", open);
+      sidenavToggle.setAttribute("aria-expanded", open ? "true" : "false");
+    });
+  }
+
+  /* ---------- article enhancements ---------- */
+  var article = document.querySelector(".ld-article");
+
+  if (article) {
+    // Language logos for code-bar labels (Simple Icons + Devicon paths, drawn in currentColor).
+    var LANG_ICONS = {
+      python: ["0 0 24 24", "M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z"],
+      rust: ["0 0 24 24", "M23.8346 11.7033l-1.0073-.6236a13.7268 13.7268 0 00-.0283-.2936l.8656-.8069a.3483.3483 0 00-.1154-.578l-1.1066-.414a8.4958 8.4958 0 00-.087-.2856l.6904-.9587a.3462.3462 0 00-.2257-.5446l-1.1663-.1894a9.3574 9.3574 0 00-.1407-.2622l.49-1.0761a.3437.3437 0 00-.0274-.3361.3486.3486 0 00-.3006-.154l-1.1845.0416a6.7444 6.7444 0 00-.1873-.2268l.2723-1.153a.3472.3472 0 00-.417-.4172l-1.1532.2724a14.0183 14.0183 0 00-.2278-.1873l.0415-1.1845a.3442.3442 0 00-.49-.328l-1.076.491c-.0872-.0476-.1742-.0952-.2623-.1407l-.1903-1.1673A.3483.3483 0 0016.256.955l-.9597.6905a8.4867 8.4867 0 00-.2855-.086l-.414-1.1066a.3483.3483 0 00-.5781-.1154l-.8069.8666a9.2936 9.2936 0 00-.2936-.0284L12.2946.1683a.3462.3462 0 00-.5892 0l-.6236 1.0073a13.7383 13.7383 0 00-.2936.0284L9.9803.3374a.3462.3462 0 00-.578.1154l-.4141 1.1065c-.0962.0274-.1903.0567-.2855.086L7.744.955a.3483.3483 0 00-.5447.2258L7.009 2.348a9.3574 9.3574 0 00-.2622.1407l-1.0762-.491a.3462.3462 0 00-.49.328l.0416 1.1845a7.9826 7.9826 0 00-.2278.1873L3.8413 3.425a.3472.3472 0 00-.4171.4171l.2713 1.1531c-.0628.075-.1255.1509-.1863.2268l-1.1845-.0415a.3462.3462 0 00-.328.49l.491 1.0761a9.167 9.167 0 00-.1407.2622l-1.1662.1894a.3483.3483 0 00-.2258.5446l.6904.9587a13.303 13.303 0 00-.087.2855l-1.1065.414a.3483.3483 0 00-.1155.5781l.8656.807a9.2936 9.2936 0 00-.0283.2935l-1.0073.6236a.3442.3442 0 000 .5892l1.0073.6236c.008.0982.0182.1964.0283.2936l-.8656.8079a.3462.3462 0 00.1155.578l1.1065.4141c.0273.0962.0567.1914.087.2855l-.6904.9587a.3452.3452 0 00.2268.5447l1.1662.1893c.0456.088.0922.1751.1408.2622l-.491 1.0762a.3462.3462 0 00.328.49l1.1834-.0415c.0618.0769.1235.1528.1873.2277l-.2713 1.1541a.3462.3462 0 00.4171.4161l1.153-.2713c.075.0638.151.1255.2279.1863l-.0415 1.1845a.3442.3442 0 00.49.327l1.0761-.49c.087.0486.1741.0951.2622.1407l.1903 1.1662a.3483.3483 0 00.5447.2268l.9587-.6904a9.299 9.299 0 00.2855.087l.414 1.1066a.3452.3452 0 00.5781.1154l.8079-.8656c.0972.0111.1954.0203.2936.0294l.6236 1.0073a.3472.3472 0 00.5892 0l.6236-1.0073c.0982-.0091.1964-.0183.2936-.0294l.8069.8656a.3483.3483 0 00.578-.1154l.4141-1.1066a8.4626 8.4626 0 00.2855-.087l.9587.6904a.3452.3452 0 00.5447-.2268l.1903-1.1662c.088-.0456.1751-.0931.2622-.1407l1.0762.49a.3472.3472 0 00.49-.327l-.0415-1.1845a6.7267 6.7267 0 00.2267-.1863l1.1531.2713a.3472.3472 0 00.4171-.416l-.2713-1.1542c.0628-.0749.1255-.1508.1863-.2278l1.1845.0415a.3442.3442 0 00.328-.49l-.49-1.076c.0475-.0872.0951-.1742.1407-.2623l1.1662-.1893a.3483.3483 0 00.2258-.5447l-.6904-.9587.087-.2855 1.1066-.414a.3462.3462 0 00.1154-.5781l-.8656-.8079c.0101-.0972.0202-.1954.0283-.2936l1.0073-.6236a.3442.3442 0 000-.5892zm-6.7413 8.3551a.7138.7138 0 01.2986-1.396.714.714 0 11-.2997 1.396zm-.3422-2.3142a.649.649 0 00-.7715.5l-.3573 1.6685c-1.1035.501-2.3285.7795-3.6193.7795a8.7368 8.7368 0 01-3.6951-.814l-.3574-1.6684a.648.648 0 00-.7714-.499l-1.473.3158a8.7216 8.7216 0 01-.7613-.898h7.1676c.081 0 .1356-.0141.1356-.088v-2.536c0-.074-.0536-.0881-.1356-.0881h-2.0966v-1.6077h2.2677c.2065 0 1.1065.0587 1.394 1.2088.0901.3533.2875 1.5044.4232 1.8729.1346.413.6833 1.2381 1.2685 1.2381h3.5716a.7492.7492 0 00.1296-.0131 8.7874 8.7874 0 01-.8119.9526zM6.8369 20.024a.714.714 0 11-.2997-1.396.714.714 0 01.2997 1.396zM4.1177 8.9972a.7137.7137 0 11-1.304.5791.7137.7137 0 011.304-.579zm-.8352 1.9813l1.5347-.6824a.65.65 0 00.33-.8585l-.3158-.7147h1.2432v5.6025H3.5669a8.7753 8.7753 0 01-.2834-3.348zm6.7343-.5437V8.7836h2.9601c.153 0 1.0792.1772 1.0792.8697 0 .575-.7107.7815-1.2948.7815zm10.7574 1.4862c0 .2187-.008.4363-.0243.651h-.9c-.09 0-.1265.0586-.1265.1477v.413c0 .973-.5487 1.1846-1.0296 1.2382-.4576.0517-.9648-.1913-1.0275-.4717-.2704-1.5186-.7198-1.8436-1.4305-2.4034.8817-.5599 1.799-1.386 1.799-2.4915 0-1.1936-.819-1.9458-1.3769-2.3153-.7825-.5163-1.6491-.6195-1.883-.6195H5.4682a8.7651 8.7651 0 014.907-2.7699l1.0974 1.151a.648.648 0 00.9182.0213l1.227-1.1743a8.7753 8.7753 0 016.0044 4.2762l-.8403 1.8982a.652.652 0 00.33.8585l1.6178.7188c.0283.2875.0425.577.0425.8717zm-9.3006-9.5993a.7128.7128 0 11.984 1.0316.7137.7137 0 01-.984-1.0316zm8.3389 6.71a.7107.7107 0 01.9395-.3625.7137.7137 0 11-.9405.3635z"],
+      java: ["0 0 128 128", "M47.617 98.12c-19.192 5.362 11.677 16.439 36.115 5.969-4.003-1.556-6.874-3.351-6.874-3.351-10.897 2.06-15.952 2.222-25.844 1.092-8.164-.935-3.397-3.71-3.397-3.71zm33.189-10.46c-14.444 2.779-22.787 2.69-33.354 1.6-8.171-.845-2.822-4.805-2.822-4.805-21.137 7.016 11.767 14.977 41.309 6.336-3.14-1.106-5.133-3.131-5.133-3.131zm11.319-60.575c.001 0-42.731 10.669-22.323 34.187 6.024 6.935-1.58 13.17-1.58 13.17s15.289-7.891 8.269-17.777c-6.559-9.215-11.587-13.793 15.634-29.58zm9.998 81.144s3.529 2.91-3.888 5.159c-14.102 4.272-58.706 5.56-71.095.171-4.45-1.938 3.899-4.625 6.526-5.192 2.739-.593 4.303-.485 4.303-.485-4.952-3.487-32.013 6.85-13.742 9.815 49.821 8.076 90.817-3.637 77.896-9.468zM85 77.896c2.395-1.634 5.703-3.053 5.703-3.053s-9.424 1.685-18.813 2.474c-11.494.964-23.823 1.154-30.012.326-14.652-1.959 8.033-7.348 8.033-7.348s-8.812-.596-19.644 4.644C17.455 81.134 61.958 83.958 85 77.896zm5.609 15.145c-.108.29-.468.616-.468.616 31.273-8.221 19.775-28.979 4.822-23.725-1.312.464-2 1.543-2 1.543s.829-.334 2.678-.72c7.559-1.575 18.389 10.119-5.032 22.286zM64.181 70.069c-4.614-10.429-20.26-19.553.007-35.559C89.459 14.563 76.492 1.587 76.492 1.587c5.23 20.608-18.451 26.833-26.999 39.667-5.821 8.745 2.857 18.142 14.688 28.815zm27.274 51.748c-19.187 3.612-42.854 3.191-56.887.874 0 0 2.874 2.38 17.646 3.331 22.476 1.437 57-.8 57.816-11.436.001 0-1.57 4.032-18.575 7.231z"],
+      bash: ["0 0 24 24", "M21.038,4.9l-7.577-4.498C13.009,0.134,12.505,0,12,0c-0.505,0-1.009,0.134-1.462,0.403L2.961,4.9 C2.057,5.437,1.5,6.429,1.5,7.503v8.995c0,1.073,0.557,2.066,1.462,2.603l7.577,4.497C10.991,23.866,11.495,24,12,24 c0.505,0,1.009-0.134,1.461-0.402l7.577-4.497c0.904-0.537,1.462-1.529,1.462-2.603V7.503C22.5,6.429,21.943,5.437,21.038,4.9z M15.17,18.946l0.013,0.646c0.001,0.078-0.05,0.167-0.111,0.198l-0.383,0.22c-0.061,0.031-0.111-0.007-0.112-0.085L14.57,19.29 c-0.328,0.136-0.66,0.169-0.872,0.084c-0.04-0.016-0.057-0.075-0.041-0.142l0.139-0.584c0.011-0.046,0.036-0.092,0.069-0.121 c0.012-0.011,0.024-0.02,0.036-0.026c0.022-0.011,0.043-0.014,0.062-0.006c0.229,0.077,0.521,0.041,0.802-0.101 c0.357-0.181,0.596-0.545,0.592-0.907c-0.003-0.328-0.181-0.465-0.613-0.468c-0.55,0.001-1.064-0.107-1.072-0.917 c-0.007-0.667,0.34-1.361,0.889-1.8l-0.007-0.652c-0.001-0.08,0.048-0.168,0.111-0.2l0.37-0.236 c0.061-0.031,0.111,0.007,0.112,0.087l0.006,0.653c0.273-0.109,0.511-0.138,0.726-0.088c0.047,0.012,0.067,0.076,0.048,0.151 l-0.144,0.578c-0.011,0.044-0.036,0.088-0.065,0.116c-0.012,0.012-0.025,0.021-0.038,0.028c-0.019,0.01-0.038,0.013-0.057,0.009 c-0.098-0.022-0.332-0.073-0.699,0.113c-0.385,0.195-0.52,0.53-0.517,0.778c0.003,0.297,0.155,0.387,0.681,0.396 c0.7,0.012,1.003,0.318,1.01,1.023C16.105,17.747,15.736,18.491,15.17,18.946z M19.143,17.859c0,0.06-0.008,0.116-0.058,0.145 l-1.916,1.164c-0.05,0.029-0.09,0.004-0.09-0.056v-0.494c0-0.06,0.037-0.093,0.087-0.122l1.887-1.129 c0.05-0.029,0.09-0.004,0.09,0.056V17.859z M20.459,6.797l-7.168,4.427c-0.894,0.523-1.553,1.109-1.553,2.187v8.833 c0,0.645,0.26,1.063,0.66,1.184c-0.131,0.023-0.264,0.039-0.398,0.039c-0.42,0-0.833-0.114-1.197-0.33L3.226,18.64 c-0.741-0.44-1.201-1.261-1.201-2.142V7.503c0-0.881,0.46-1.702,1.201-2.142l7.577-4.498c0.363-0.216,0.777-0.33,1.197-0.33 c0.419,0,0.833,0.114,1.197,0.33l7.577,4.498c0.624,0.371,1.046,1.013,1.164,1.732C21.686,6.557,21.12,6.411,20.459,6.797z"],
+    };
+    var LANG_ALIASES = { py: "python", python3: "python", rs: "rust", sh: "bash", shell: "bash", console: "bash", zsh: "bash" };
+
+    // Code blocks: wrap .highlight in a window with a language bar + copy button.
+    article.querySelectorAll("div.highlight").forEach(function (hl) {
+      if (hl.closest(".ld-code")) return;
+      var code = hl.querySelector("code");
+      var m = (hl.className + " " + (code ? code.className : "")).match(/language-([\w+-]+)/);
+      var lang = m ? m[1] : "text";
+      var wrap = document.createElement("div");
+      wrap.className = "ld-code";
+      var bar = document.createElement("div");
+      bar.className = "ld-code__bar";
+      bar.innerHTML = "<span></span><button class=\"ld-copy\" type=\"button\">Copy</button>";
+      var span = bar.querySelector("span");
+      var icon = LANG_ICONS[LANG_ALIASES[lang] || lang];
+      if (icon) span.innerHTML = '<svg viewBox="' + icon[0] + '" fill="currentColor" aria-hidden="true"><path d="' + icon[1] + '"/></svg>';
+      span.appendChild(document.createTextNode(lang));
+      hl.parentNode.insertBefore(wrap, hl);
+      wrap.appendChild(bar);
+      wrap.appendChild(hl);
+    });
+
+    document.addEventListener("click", function (e) {
+      var copy = e.target.closest(".ld-copy");
+      if (!copy) return;
+      var pre = copy.closest(".ld-code").querySelector("pre");
+      if (navigator.clipboard && pre) navigator.clipboard.writeText(pre.textContent);
+      copy.textContent = "Copied";
+      setTimeout(function () { copy.textContent = "Copy"; }, 1400);
+    });
+
+    // Tables: wrap for horizontal overflow.
+    article.querySelectorAll("table").forEach(function (t) {
+      if (t.closest(".ld-tablewrap")) return;
+      var wrap = document.createElement("div");
+      wrap.className = "ld-tablewrap";
+      t.parentNode.insertBefore(wrap, t);
+      wrap.appendChild(t);
+    });
+
+    // Mermaid: render fenced diagrams on demand.
+    var mermaidNodes = article.querySelectorAll("pre.mermaid, div.mermaid");
+    if (mermaidNodes.length) {
+      import("https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs").then(function (mod) {
+        var mermaid = mod.default;
+        mermaidNodes.forEach(function (node) {
+          if (node.tagName === "PRE") {
+            var div = document.createElement("div");
+            div.className = "mermaid";
+            div.textContent = node.textContent;
+            node.replaceWith(div);
+          }
+        });
+        mermaid.initialize({ startOnLoad: false, securityLevel: "loose", theme: "neutral" });
+        mermaid.run({ querySelector: ".ld-article div.mermaid" });
+      }).catch(function () { /* offline: leave the diagram source visible */ });
+    }
+  }
+
+  /* ---------- TOC scroll-spy ---------- */
+  var tocLinks = Array.prototype.slice.call(document.querySelectorAll(".ld-toc a"));
+  if (tocLinks.length && article) {
+    var targets = tocLinks.map(function (a) {
+      var id = decodeURIComponent((a.getAttribute("href") || "").replace(/^#/, ""));
+      return document.getElementById(id);
+    });
+    var update = function () {
+      var active = 0;
+      for (var i = 0; i < targets.length; i++) {
+        if (targets[i] && targets[i].getBoundingClientRect().top <= 90) active = i;
+      }
+      tocLinks.forEach(function (a, i) { a.classList.toggle("active", i === active); });
+    };
+    window.addEventListener("scroll", update, { passive: true });
+    update();
+  }
+
+  /* ---------- search overlay ---------- */
+  var overlay = document.getElementById("search-overlay");
+  var input = document.getElementById("search-input");
+  var results = document.getElementById("search-results");
+  var indexPromise = null;
+
+  function loadIndex() {
+    if (!indexPromise) {
+      indexPromise = fetch(BASE + "/search/search_index.json")
+        .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+        .then(function (d) {
+          return d.docs.map(function (doc) {
+            return {
+              location: doc.location,
+              title: doc.title || "",
+              text: (doc.text || "").replace(/\s+/g, " "),
+            };
+          });
+        });
+    }
+    return indexPromise;
+  }
+
+  function esc(s) {
+    return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  }
+
+  function highlight(text, terms) {
+    var out = esc(text);
+    terms.forEach(function (t) {
+      out = out.replace(new RegExp("(" + t.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ")", "ig"), "<mark>$1</mark>");
+    });
+    return out;
+  }
+
+  function search(docs, query) {
+    var terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    if (!terms.length) return [];
+    var scored = [];
+    docs.forEach(function (doc) {
+      var title = doc.title.toLowerCase();
+      var text = doc.text.toLowerCase();
+      var score = 0;
+      for (var i = 0; i < terms.length; i++) {
+        var t = terms[i];
+        var inTitle = title.indexOf(t) !== -1;
+        var inText = text.indexOf(t) !== -1;
+        if (!inTitle && !inText) { score = 0; break; }
+        score += (inTitle ? 10 : 0) + (inText ? 1 : 0);
+      }
+      // Prefer page-level entries slightly over deep anchors.
+      if (score > 0) scored.push({ doc: doc, score: score + (doc.location.indexOf("#") === -1 ? 2 : 0) });
+    });
+    scored.sort(function (a, b) { return b.score - a.score; });
+    return scored.slice(0, 12).map(function (s) { return s.doc; });
+  }
+
+  function snippet(text, terms) {
+    var lower = text.toLowerCase();
+    var pos = -1;
+    for (var i = 0; i < terms.length; i++) {
+      pos = lower.indexOf(terms[i]);
+      if (pos !== -1) break;
+    }
+    if (pos === -1) pos = 0;
+    var start = Math.max(0, pos - 60);
+    var s = (start > 0 ? "…" : "") + text.slice(start, start + 160) + (start + 160 < text.length ? "…" : "");
+    return s;
+  }
+
+  function render(docs, query) {
+    var terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    if (!docs.length) {
+      results.innerHTML = "<div class=\"ld-search__empty\">No results</div>";
+      return;
+    }
+    results.innerHTML = docs.map(function (doc) {
+      var crumb = doc.location.split("#")[0].replace(/\/$/, "").replace(/\//g, " / ") || "home";
+      return "<a class=\"ld-search__hit\" href=\"" + BASE + "/" + doc.location + "\">" +
+        "<div class=\"ld-search__hit-crumb\">" + esc(crumb) + "</div>" +
+        "<div class=\"ld-search__hit-title\">" + highlight(doc.title, terms) + "</div>" +
+        "<div class=\"ld-search__hit-text\">" + highlight(snippet(doc.text, terms), terms) + "</div>" +
+        "</a>";
+    }).join("");
+  }
+
+  function openSearch() {
+    overlay.hidden = false;
+    input.value = "";
+    results.innerHTML = "";
+    input.focus();
+    loadIndex();
+  }
+  function closeSearch() { overlay.hidden = true; }
+
+  if (overlay && input && results) {
+    var openBtn = document.getElementById("search-open");
+    if (openBtn) openBtn.addEventListener("click", openSearch);
+    overlay.addEventListener("click", function (e) {
+      if (e.target.closest("[data-search-close]")) closeSearch();
+    });
+    document.addEventListener("keydown", function (e) {
+      if (e.key === "Escape" && !overlay.hidden) { closeSearch(); return; }
+      var typing = /^(INPUT|TEXTAREA|SELECT)$/.test((document.activeElement || {}).tagName || "");
+      if ((e.key === "/" && !typing) || ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k")) {
+        e.preventDefault();
+        if (overlay.hidden) openSearch(); else closeSearch();
+      }
+    });
+    var pending = 0;
+    input.addEventListener("input", function () {
+      var q = input.value.trim();
+      var seq = ++pending;
+      if (q.length < 2) { results.innerHTML = ""; return; }
+      loadIndex().then(function (docs) {
+        if (seq !== pending) return;
+        render(search(docs, q), q);
+      }).catch(function () {
+        results.innerHTML = "<div class=\"ld-search__empty\">Search index unavailable</div>";
+      });
+    });
+  }
+})();
diff --git a/docs/theme/assets/tokens.css b/docs/theme/assets/tokens.css
new file mode 100644
index 00000000000..29c08f2711e
--- /dev/null
+++ b/docs/theme/assets/tokens.css
@@ -0,0 +1,142 @@
+/* Lance Design System — tokens (Swiss editorial: white, ink, brand purple #625EFF).
+   Merged from the design project's ds/tokens set; dark theme mirrors the same
+   structure with paper rules on ink. */
+@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600&family=Roboto+Mono:wght@400;500;600&display=swap");
+
+:root {
+  color-scheme: light;
+
+  /* ---- Typography ---- */
+  --font-display: "Space Grotesk", "Segoe UI", sans-serif;
+  --font-body: "IBM Plex Sans", "Segoe UI", sans-serif;
+  --font-mono: "Roboto Mono", "SF Mono", monospace;
+  --text-base: 16px;
+  --leading-body: 1.6;
+  --tracking-display: -0.045em;
+  --tracking-caps: 0.14em;
+
+  /* ---- Ink scale (dark values — code surfaces & tooltips only) ---- */
+  --ink-950: #0E0C14;
+  --ink-900: #14111D;
+  --ink-850: #1A1626;
+  --ink-800: #221D30;
+  --ink-700: #2C2640;
+  --ink-600: #3A3352;
+
+  /* ---- Foreground scale (on white) ---- */
+  --fg-1: #0E0C14;
+  --fg-2: #4A4459;
+  --fg-3: #8A8499;
+  --fg-inverse: #ffffff;
+
+  /* ---- Brand purple ramp (official #625EFF) ---- */
+  --beam-300: #B3B1FF;
+  --beam-400: #625EFF;
+  --beam-500: #625EFF;
+  --beam-600: #4B47E0;
+  --beam-700: #3936B4;
+  --beam-glow: rgba(98, 94, 255, 0.25);
+  --beam-dim: rgba(98, 94, 255, 0.09);
+
+  /* ---- Semantic status ---- */
+  --ok-500: #0E8F63;
+  --warn-500: #B26205;
+  --danger-500: #D92D20;
+
+  /* ---- Lines: ink rules carry the structure ---- */
+  --line-1: #14111D;
+  --line-2: rgba(20, 17, 29, 0.16);
+
+  /* ---- Surfaces ---- */
+  --surface-page: #FFFFFF;
+  --surface-card: #FFFFFF;
+  --surface-overlay: #F5F4F8;
+  --surface-header: rgba(255, 255, 255, 0.88);
+  --surface-code: #F5F4F8; /* light code surface */
+  --surface-inline-code: rgba(20, 17, 29, 0.05);
+
+  --text-body: var(--fg-1);
+  --text-secondary: var(--fg-2);
+  --text-muted: var(--fg-3);
+
+  /* ---- Syntax highlighting (Pygments), light theme ---- */
+  --code-fg: #2A2635;
+  --code-kw: #7C3AED;     /* keywords, builtins */
+  --code-str: #0B62C4;    /* strings */
+  --code-com: #6E6A80;    /* comments */
+  --code-num: #9A6700;    /* numbers */
+  --code-fn: #0F766E;     /* functions, decorators */
+  --code-name: #2A2635;   /* class/module names, headings */
+  --code-tag: #BE123C;    /* HTML tags, attributes */
+  --code-punct: #57534E;  /* operators, punctuation, output */
+  --code-prompt: #8A8499; /* REPL prompts */
+  --code-hll: rgba(98, 94, 255, 0.12); /* highlighted line */
+
+  /* ---- Layout ---- */
+  --container-max: 1200px;
+  --radius-box: 6px;   /* content boxes: code, admonitions, tabs */
+  --radius-chip: 4px;  /* inline code chips */
+
+  /* ---- Motion — fast, precise, no bounce ---- */
+  --ease-out: cubic-bezier(0.2, 0.8, 0.2, 1);
+  --dur-fast: 120ms;
+}
+
+/* ---- Dark theme: same structure, paper rules on ink ---- */
+:root[data-theme="dark"] {
+  color-scheme: dark;
+
+  --fg-1: #F2F0FA;
+  --fg-2: #B7B1C6;
+  --fg-3: #837D96;
+  --fg-inverse: #0E0C14;
+
+  /* text accents need a lighter purple to stay readable on ink */
+  --beam-600: #8E8BFF;
+  --beam-700: #A9A6FF;
+
+  --ok-500: #34C08E;
+  --warn-500: #E8A13C;
+  --danger-500: #FF6B5E;
+
+  --line-1: rgba(237, 235, 245, 0.7);
+  --line-2: rgba(237, 235, 245, 0.15);
+
+  --surface-page: #0E0C14;
+  --surface-card: #14111D;
+  --surface-overlay: #1A1626;
+  --surface-header: rgba(14, 12, 20, 0.85);
+  --surface-code: #14111D;
+  --surface-inline-code: rgba(237, 235, 245, 0.08);
+
+  /* ---- Syntax highlighting (Pygments), dark theme (on ink) ---- */
+  --code-fg: #EDEBF5;
+  --code-kw: #B8A8FF;
+  --code-str: #7CC0FF;
+  --code-com: #6F6A85;
+  --code-num: #F2CE6B;
+  --code-fn: #8FE8CE;
+  --code-name: #EDEBF5;
+  --code-tag: #FF95A8;
+  --code-punct: #B7B1C6;
+  --code-prompt: #6F6A85;
+  --code-hll: rgba(98, 94, 255, 0.22);
+}
+
+/* ---- Minimal base ---- */
+* { box-sizing: border-box; }
+
+body {
+  margin: 0;
+  background: var(--surface-page);
+  color: var(--text-body);
+  font-family: var(--font-body);
+  font-size: var(--text-base);
+  line-height: var(--leading-body);
+  -webkit-font-smoothing: antialiased;
+}
+
+/* No `color` here: forcing ink text breaks selection inside dark code blocks. */
+::selection { background: rgba(98, 94, 255, 0.3); }
+
+code, kbd, pre { font-family: var(--font-mono); }
diff --git a/docs/theme/base.html b/docs/theme/base.html
new file mode 100644
index 00000000000..d90b2ae02c4
--- /dev/null
+++ b/docs/theme/base.html
@@ -0,0 +1,118 @@
+{#- Lance Docs custom theme — shared skeleton: head, header, footer, scripts. -#}
+{%- macro first_url(item) -%}
+  {%- if item.is_link or item.is_page -%}
+    {{- item.url -}}
+  {%- elif item.children -%}
+    {{- first_url(item.children[0]) -}}
+  {%- endif -%}
+{%- endmacro -%}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+{% if page and page.title and not page.is_homepage %}
+<title>{{ page.title }} — {{ config.site_name }}</title>
+{% else %}
+<title>{{ config.site_name }} — The open lakehouse format for multimodal AI</title>
+{% endif %}
+{% if page and page.meta and page.meta.description %}
+<meta name="description" content="{{ page.meta.description }}">
+{% elif config.site_description %}
+<meta name="description" content="{{ config.site_description }}">
+{% endif %}
+{% if page and page.canonical_url %}<link rel="canonical" href="{{ page.canonical_url }}">{% endif %}
+<link rel="icon" href="{{ 'logo/logo.png' | url }}">
+<link rel="stylesheet" href="{{ 'assets/tokens.css' | url }}">
+<link rel="stylesheet" href="{{ 'assets/site.css' | url }}">
+<script>
+/* Set the theme before first paint to avoid a light-mode flash. */
+(function () {
+  try {
+    var t = localStorage.getItem("ld-theme");
+    if (t !== "light" && t !== "dark") {
+      t = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
+    }
+    document.documentElement.dataset.theme = t;
+  } catch (e) { document.documentElement.dataset.theme = "light"; }
+})();
+</script>
+</head>
+<body>
+<div class="ld-shell">
+  <header class="ld-header">
+    <a class="ld-brand" href="{{ nav.homepage.url | url }}" aria-label="{{ config.site_name }}">
+      <img src="{{ 'logo/wide.png' | url }}" alt="{{ config.site_name }}">
+    </a>
+    <nav class="ld-topnav" aria-label="Sections">
+      {%- for item in nav %}
+      <a class="ld-toptab{% if item.active %} active{% endif %}" href="{{ first_url(item) | url }}">{{ item.title }}</a>
+      {%- endfor %}
+    </nav>
+    <div class="ld-header__actions">
+      <button class="ld-btn ld-btn--ghost ld-btn--icon" id="search-open" type="button" aria-label="Search docs" title="Search (/)">
+        <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><circle cx="11" cy="11" r="7"/><path d="m21 21-4.3-4.3"/></svg>
+      </button>
+      <a class="ld-btn ld-btn--ghost" href="{{ config.repo_url }}" target="_blank" rel="noopener" aria-label="Lance on GitHub">
+        <svg viewBox="0 0 16 16" width="14" height="14" fill="currentColor" aria-hidden="true"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82a7.4 7.4 0 0 1 2-.27c.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58-8-8-8Z"/></svg>
+        <span class="ld-btn__label">GitHub</span>
+        <span class="ld-btn__count" id="gh-stars" hidden></span>
+      </a>
+      <a class="ld-btn ld-btn--ghost ld-btn--icon" href="https://discord.gg/lance" target="_blank" rel="noopener" aria-label="Join the Lance Discord" title="Discord">
+        <svg viewBox="0 0 24 24" width="15" height="15" fill="currentColor" aria-hidden="true"><path d="M20.32 4.37a19.8 19.8 0 0 0-4.89-1.52.07.07 0 0 0-.08.04c-.21.38-.44.87-.61 1.25a18.27 18.27 0 0 0-5.48 0 12.64 12.64 0 0 0-.62-1.25.08.08 0 0 0-.08-.04c-1.71.3-3.35.81-4.88 1.52a.07.07 0 0 0-.04.03C.53 9.05-.32 13.58.1 18.06c0 .02.01.04.03.05a19.9 19.9 0 0 0 6 3.03c.03.01.06 0 .08-.02.46-.63.87-1.3 1.22-2a.08.08 0 0 0-.04-.11 13.1 13.1 0 0 1-1.87-.89.08.08 0 0 1-.01-.13c.13-.09.25-.19.37-.29a.07.07 0 0 1 .08-.01c3.93 1.79 8.18 1.79 12.06 0a.07.07 0 0 1 .08.01c.12.1.25.2.37.29a.08.08 0 0 1-.01.13c-.6.35-1.22.64-1.87.89a.08.08 0 0 0-.04.11c.36.7.77 1.36 1.22 2 .02.02.05.03.08.02a19.84 19.84 0 0 0 6.02-3.03.08.08 0 0 0 .03-.05c.5-5.18-.84-9.68-3.55-13.66a.06.06 0 0 0-.03-.03ZM8.02 15.33c-1.18 0-2.16-1.08-2.16-2.42 0-1.33.96-2.42 2.16-2.42 1.21 0 2.18 1.1 2.16 2.42 0 1.34-.96 2.42-2.16 2.42Zm7.97 0c-1.18 0-2.15-1.08-2.15-2.42 0-1.33.95-2.42 2.15-2.42 1.21 0 2.18 1.1 2.16 2.42 0 1.34-.95 2.42-2.16 2.42Z"/></svg>
+      </a>
+      <button class="ld-btn ld-btn--ghost ld-btn--icon" id="theme-toggle" type="button" aria-label="Toggle light / dark theme" title="Toggle theme">
+        <svg class="ld-icon-sun" viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v2m0 16v2M4.93 4.93l1.41 1.41m11.32 11.32 1.41 1.41M2 12h2m16 0h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"/></svg>
+        <svg class="ld-icon-moon" viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79Z"/></svg>
+      </button>
+      <a class="ld-btn ld-btn--primary" href="{{ 'quickstart/' | url }}">Get started</a>
+    </div>
+  </header>
+
+  {% block container %}{% endblock %}
+
+  <footer class="ld-footer">
+    <div class="ld-footer__brand">
+      <img src="{{ 'logo/wide.png' | url }}" alt="{{ config.site_name }}">
+      <p>Lance is an open-source columnar format for multimodal AI. Apache-2.0. {{ config.copyright }}</p>
+    </div>
+    <div class="ld-footer__cols">
+      <div class="ld-footer__col">
+        <h5>Format</h5>
+        <a href="{{ 'format/' | url }}">Specification</a>
+        <a href="{{ 'format/file/' | url }}">File format</a>
+        <a href="{{ 'format/table/' | url }}">Table format</a>
+      </div>
+      <div class="ld-footer__col">
+        <h5>Develop</h5>
+        <a href="{{ 'quickstart/' | url }}">Getting started</a>
+        <a href="{{ 'guide/read_and_write/' | url }}">User guide</a>
+        <a href="{{ 'sdk_docs/' | url }}">SDK docs</a>
+      </div>
+      <div class="ld-footer__col">
+        <h5>Community</h5>
+        <a href="{{ config.repo_url }}" target="_blank" rel="noopener">GitHub</a>
+        <a href="https://discord.gg/lance" target="_blank" rel="noopener">Discord</a>
+        <a href="https://groups.google.com/a/lance.org/g/dev" target="_blank" rel="noopener">Mailing list</a>
+        <a href="{{ 'community/contributing/' | url }}">Contributing</a>
+      </div>
+    </div>
+  </footer>
+</div>
+
+<!-- Search overlay -->
+<div class="ld-search" id="search-overlay" hidden>
+  <div class="ld-search__scrim" data-search-close></div>
+  <div class="ld-search__panel" role="dialog" aria-label="Search documentation">
+    <div class="ld-search__bar">
+      <input class="ld-search__input" id="search-input" type="search" placeholder="Search documentation…" autocomplete="off" spellcheck="false">
+      <button class="ld-search__close" type="button" data-search-close aria-label="Close search">Esc</button>
+    </div>
+    <div class="ld-search__results" id="search-results"></div>
+  </div>
+</div>
+
+<script>window.LD_BASE = "{{ base_url }}";</script>
+<script src="{{ 'assets/site.js' | url }}" defer></script>
+</body>
+</html>
diff --git a/docs/theme/home.html b/docs/theme/home.html
new file mode 100644
index 00000000000..1b361168d46
--- /dev/null
+++ b/docs/theme/home.html
@@ -0,0 +1,147 @@
+{% extends "base.html" %}
+
+{% block container %}
+  <main class="ld-home">
+    <section class="ld-hero">
+      <div class="ld-kicker"><span>Lance format</span> · <span>Open source</span> · <span>Apache-2.0</span> · <a href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">VLDB '25 paper ↗</a></div>
+      <h1>The open lakehouse format for multimodal AI.</h1>
+      <p class="ld-hero__lead">A file format, table format, and catalog spec for building a complete lakehouse on object storage — powering vector and full-text search, feature engineering, and model training with the fast random access and scans that AI workloads need.</p>
+      <div class="ld-hero__cta">
+        <a class="ld-btn ld-btn--primary ld-btn--lg" href="{{ 'quickstart/' | url }}">Get started</a>
+        <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'format/' | url }}">Read the spec</a>
+        <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'examples/python/llm_training/' | url }}">Train an LLM</a>
+        <a class="ld-btn ld-btn--text ld-btn--lg" href="https://discord.gg/lance" target="_blank" rel="noopener">Join Discord →</a>
+      </div>
+    </section>
+
+    <section class="ld-band">
+      <div class="ld-stats">
+        <div class="ld-stat">
+          <div class="ld-stat__value">100×</div>
+          <div class="ld-stat__label">Faster random access than Parquet</div>
+        </div>
+        <div class="ld-stat">
+          <div class="ld-stat__value">1 line</div>
+          <div class="ld-stat__label">To convert Parquet to Lance</div>
+        </div>
+        <a class="ld-stat ld-stat--link" href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">
+          <div class="ld-stat__value ld-stat__value--beam">VLDB '25</div>
+          <div class="ld-stat__label">Peer-reviewed research paper →</div>
+        </a>
+      </div>
+    </section>
+
+    <section class="ld-what">
+      <h2>What is Lance?</h2>
+      <p class="ld-what__body">Lance is a modern, open source lakehouse format for multimodal AI. It brings high-performance vector and full-text search, feature engineering, and model training to the lakehouse, powered by fast random access and scans — while keeping SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB) and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore).</p>
+      <p class="ld-what__more">Learn more in the <a class="ld-inline-link" href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">research paper</a> published at VLDB 2025.</p>
+    </section>
+
+    <section class="ld-features">
+      <div class="ld-feature">
+        <div class="ld-feature__num">01</div>
+        <div class="ld-feature__body">
+          <h3>Expressive hybrid search</h3>
+          <p>Combine vector similarity, full-text search (BM25), and SQL analytics on the same dataset. All query types are accelerated by secondary indexes that are part of the Lance specification.</p>
+          <a class="ld-more" href="{{ 'quickstart/vector-search/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/docs"</span>)
+
+<span class="tok-com"># Full text search</span>
+ds.to_table(<span class="tok-arg">full_text_query=</span><span class="tok-str">"machine learning"</span>)
+
+<span class="tok-com"># Hybrid search</span>
+ds.to_table(
+    <span class="tok-arg">nearest=</span>{
+        <span class="tok-str">"column"</span>: <span class="tok-str">"embedding"</span>, <span class="tok-str">"q"</span>: query_vec, <span class="tok-str">"k"</span>: <span class="tok-num">10</span>
+    },
+    <span class="tok-arg">filter=</span><span class="tok-str">"year &gt; 2020"</span>,
+)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">02</div>
+        <div class="ld-feature__body">
+          <h3>Lightning-fast random access</h3>
+          <p>100x faster random access than Parquet or Iceberg. An optimized file format plus row addressing and secondary indexes let you fetch individual records across files instantly — for ML serving, sampling, and interactive apps.</p>
+          <a class="ld-more" href="{{ 'guide/read_and_write/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/embeddings.lance"</span>)
+
+<span class="tok-com"># Access the 2nd &amp; 51st rows</span>
+ds.take([<span class="tok-num">2</span>, <span class="tok-num">51</span>], <span class="tok-arg">columns=</span>[<span class="tok-str">"id"</span>, <span class="tok-str">"vec_gemma3"</span>])
+
+<span class="tok-com"># Take 1000 random samples</span>
+ds.sample(<span class="tok-num">1000</span>, <span class="tok-arg">columns=</span>[<span class="tok-str">"id"</span>, <span class="tok-str">"vec_llama"</span>])</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">03</div>
+        <div class="ld-feature__body">
+          <h3>Native multimodal data</h3>
+          <p>Store images, videos, audio, text, and embeddings alongside tabular data in one format. Blob encoding handles large binary objects with lazy loading; optimized vector storage accelerates similarity search.</p>
+          <a class="ld-more" href="{{ 'guide/blob/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+<span class="tok-kw">import</span> av
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/videos.lance"</span>)
+
+<span class="tok-com"># Get blobs from the 2nd and 51st rows</span>
+blobs = ds.take_blobs(<span class="tok-str">"video"</span>, <span class="tok-arg">ids=</span>[<span class="tok-num">2</span>, <span class="tok-num">51</span>])
+
+<span class="tok-kw">for</span> blob <span class="tok-kw">in</span> blobs:
+    <span class="tok-kw">with</span> av.open(blob) <span class="tok-kw">as</span> container:
+        stream = container.streams.video[<span class="tok-num">0</span>]
+        container.seek(<span class="tok-arg">start_time=</span><span class="tok-num">500</span>, <span class="tok-arg">stream=</span>stream)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">04</div>
+        <div class="ld-feature__body">
+          <h3>Data evolution &gt; schema evolution</h3>
+          <p>Backfilling column values normally forces a full table rewrite. Lance supports efficient schema evolution with backfill — adding a column with data is just writing new Lance files to the table.</p>
+          <a class="ld-more" href="{{ 'guide/data_evolution/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+dataset = lance.dataset(<span class="tok-str">"my_data.lance"</span>)
+
+<span class="tok-fn">@lance.batch_udf</span>()
+<span class="tok-kw">def</span> <span class="tok-fn">add_embeddings</span>(batch):
+    vectors = model.encode(batch[<span class="tok-str">"text"</span>])
+    <span class="tok-kw">return</span> {<span class="tok-str">"embedding"</span>: vectors}
+
+dataset.add_columns(add_embeddings)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">05</div>
+        <div class="ld-feature__body">
+          <h3>Rich ecosystem integrations</h3>
+          <p>Works with Pandas, Polars, Ray, and PyTorch for processing and ML. Connects to Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink for SQL analytics and distributed processing.</p>
+          <a class="ld-more" href="{{ 'integrations/' | url }}">View integrations →</a>
+        </div>
+        <div class="ld-feature__img">
+          <img src="{{ 'assets/images/ecosystem-integrations.png' | url }}" alt="Lance ecosystem integrations" loading="lazy">
+        </div>
+      </div>
+    </section>
+  </main>
+{% endblock %}
diff --git a/docs/theme/main.html b/docs/theme/main.html
new file mode 100644
index 00000000000..f9a4a7e6bd9
--- /dev/null
+++ b/docs/theme/main.html
@@ -0,0 +1,92 @@
+{% extends "base.html" %}
+
+{#- Render one nav section as sidenav groups: its direct pages/links under the
+    section's own label, then each child section as a further group. -#}
+{%- macro sidenav_section(sec, label) -%}
+  {%- set direct = sec.children | selectattr("is_section", "ne", true) | list -%}
+  {%- if direct -%}
+  <div class="ld-sidenav__group">
+    <div class="ld-sidenav__label">{{ label }}</div>
+    {%- for child in direct %}
+    {%- if child.is_link %}
+    <a href="{{ child.url }}" target="_blank" rel="noopener">{{ child.title }} <span class="ext">↗</span></a>
+    {%- else %}
+    <a{% if child == page %} class="active"{% endif %} href="{{ child.url | url }}">{{ child.title }}</a>
+    {%- endif %}
+    {%- endfor %}
+  </div>
+  {%- endif -%}
+  {%- for child in sec.children if child.is_section -%}
+    {{ sidenav_section(child, child.title) }}
+  {%- endfor -%}
+{%- endmacro -%}
+
+{% block container %}
+  {%- set active_section = namespace(item=none) -%}
+  {%- for item in nav %}{% if item.active %}{% set active_section.item = item %}{% endif %}{% endfor %}
+
+  <main class="ld-docs">
+    <button class="ld-sidenav-toggle" type="button" aria-expanded="false" aria-controls="ld-sidenav">
+      {{ active_section.item.title if active_section.item else "Menu" }}
+      <svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><path d="m6 9 6 6 6-6"/></svg>
+    </button>
+    <aside class="ld-sidenav" id="ld-sidenav" aria-label="Section pages">
+      {%- if active_section.item and active_section.item.is_section %}
+        {{ sidenav_section(active_section.item, active_section.item.title) }}
+      {%- elif active_section.item %}
+      <div class="ld-sidenav__group">
+        <div class="ld-sidenav__label">{{ active_section.item.title }}</div>
+        <a class="active" href="{{ active_section.item.url | url }}">{{ active_section.item.title }}</a>
+      </div>
+      {%- endif %}
+    </aside>
+
+    <div class="ld-article-col">
+      <div class="ld-crumbs">
+        <span>Docs</span>
+        {%- for crumb in page.ancestors | reverse %}
+        <span>/</span><span>{{ crumb.title }}</span>
+        {%- endfor %}
+        <span>/</span><b>{{ page.title }}</b>
+      </div>
+
+      <article class="ld-article">
+        {{ page.content }}
+      </article>
+
+      <div class="ld-pagenav">
+        {%- if page.previous_page %}
+        <a href="{{ page.previous_page.url | url }}">
+          <div class="ld-pagenav__k">← Previous</div>
+          <div class="ld-pagenav__t">{{ page.previous_page.title }}</div>
+        </a>
+        {%- else %}<span></span>{% endif %}
+        {%- if page.next_page %}
+        <a class="next" href="{{ page.next_page.url | url }}">
+          <div class="ld-pagenav__k">Next →</div>
+          <div class="ld-pagenav__t">{{ page.next_page.title }}</div>
+        </a>
+        {%- endif %}
+      </div>
+    </div>
+
+    <aside class="ld-toc" aria-label="On this page">
+      {%- set toc_items = namespace(list=[]) -%}
+      {%- for h1 in page.toc -%}
+        {%- if loop.first and page.toc | length == 1 -%}
+          {%- set toc_items.list = h1.children -%}
+        {%- endif -%}
+      {%- endfor -%}
+      {%- if page.toc | length > 1 %}{% set toc_items.list = page.toc %}{% endif -%}
+      {%- if toc_items.list %}
+      <div class="ld-toc__label">On this page</div>
+      {%- for h2 in toc_items.list %}
+      <a class="ld-toc__h2" href="{{ h2.url }}">{{ h2.title }}</a>
+      {%- for h3 in h2.children %}
+      <a class="ld-toc__h3" href="{{ h3.url }}">{{ h3.title }}</a>
+      {%- endfor %}
+      {%- endfor %}
+      {%- endif %}
+    </aside>
+  </main>
+{% endblock %}
diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock
index e392fd82c76..68f0528b0eb 100644
--- a/java/lance-jni/Cargo.lock
+++ b/java/lance-jni/Cargo.lock
@@ -145,9 +145,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -166,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -180,9 +180,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -199,9 +199,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
 dependencies = [
  "bytes",
  "half",
@@ -211,9 +211,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -233,9 +233,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -248,9 +248,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -261,9 +261,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -277,9 +277,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -302,9 +302,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -315,9 +315,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -328,9 +328,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
 dependencies = [
  "bitflags",
  "serde_core",
@@ -339,9 +339,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -353,9 +353,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -411,18 +411,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -1008,9 +1008,23 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
 
 [[package]]
 name = "byteorder"
@@ -1020,9 +1034,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1045,9 +1059,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.65"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1126,9 +1140,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1136,9 +1150,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1148,14 +1162,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -1385,18 +1399,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1518,7 +1532,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -1529,7 +1543,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -1548,14 +1562,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -1582,12 +1595,11 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -1597,9 +1609,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1622,9 +1634,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1645,32 +1657,33 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
- "paste",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
 dependencies = [
  "futures",
  "log",
@@ -1679,9 +1692,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1701,16 +1714,17 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.4",
+ "parking_lot",
+ "rand 0.9.5",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1732,9 +1746,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1755,9 +1769,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1772,27 +1786,25 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -1801,18 +1813,19 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -1823,29 +1836,27 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1860,26 +1871,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -1889,19 +1899,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -1910,9 +1919,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -1926,34 +1935,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1964,14 +1973,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -1979,20 +1987,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
 dependencies = [
  "arrow",
  "chrono",
@@ -2009,11 +2017,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2021,20 +2028,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2047,26 +2053,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2082,12 +2088,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2102,7 +2109,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2114,9 +2121,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2125,15 +2132,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2145,9 +2151,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2163,9 +2169,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2255,7 +2261,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -2470,10 +2476,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -2547,7 +2553,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -2698,12 +2704,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -2755,11 +2762,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if 1.0.4",
- "js-sys",
  "libc",
  "r-efi 5.3.0",
  "wasip2",
- "wasm-bindgen",
 ]
 
 [[package]]
@@ -2793,7 +2798,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -2826,7 +2831,7 @@ dependencies = [
  "hostname",
  "prost",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "serde",
  "thiserror 2.0.18",
@@ -2863,6 +2868,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -2915,6 +2921,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3495,7 +3506,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -3556,7 +3567,7 @@ dependencies = [
  "quote",
  "rustc_version",
  "simd_cesu8",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -3584,7 +3595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
 dependencies = [
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -3622,7 +3633,7 @@ dependencies = [
  "nom",
  "num-traits",
  "ordered-float",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "serde_json",
  "zmij",
@@ -3647,7 +3658,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -3702,7 +3713,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
  "roaring",
  "rustc-hash",
@@ -3720,7 +3731,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3729,13 +3740,14 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -3762,7 +3774,7 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -3772,7 +3784,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3795,7 +3807,7 @@ dependencies = [
  "object_store",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "roaring",
  "serde_json",
  "snafu",
@@ -3810,7 +3822,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3842,7 +3854,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3852,23 +3864,23 @@ dependencies = [
  "futures",
  "half",
  "hex",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rand_xoshiro",
 ]
 
 [[package]]
 name = "lance-derive"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3893,7 +3905,7 @@ dependencies = [
  "num-traits",
  "prost",
  "prost-build",
- "rand 0.9.4",
+ "rand 0.9.5",
  "strum",
  "tokio",
  "tracing",
@@ -3903,7 +3915,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3933,7 +3945,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -3947,12 +3959,13 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arc-swap",
  "arrow",
  "arrow-arith",
  "arrow-array",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -3985,6 +3998,7 @@ dependencies = [
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
@@ -3998,7 +4012,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rangemap",
  "rayon",
@@ -4013,9 +4027,31 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "10.0.0-beta.3"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4033,6 +4069,7 @@ dependencies = [
  "bytes",
  "chrono",
  "futures",
+ "goosefs-sdk",
  "http 1.4.2",
  "io-uring",
  "lance-arrow",
@@ -4046,7 +4083,7 @@ dependencies = [
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "tempfile",
  "tokio",
@@ -4056,7 +4093,7 @@ dependencies = [
 
 [[package]]
 name = "lance-jni"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4092,7 +4129,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4102,13 +4139,13 @@ dependencies = [
  "lance-arrow",
  "lance-core",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4120,7 +4157,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4140,7 +4177,7 @@ dependencies = [
  "lance-table",
  "log",
  "object_store",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "roaring",
  "serde",
@@ -4169,7 +4206,7 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4184,7 +4221,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4206,7 +4243,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rangemap",
  "roaring",
  "semver",
@@ -4221,7 +4258,7 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "icu_segmenter",
  "rust-stemmers",
@@ -4298,9 +4335,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libm"
@@ -4686,7 +4723,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -5279,7 +5316,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -5386,7 +5423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -5432,7 +5469,7 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn",
+ "syn 2.0.118",
  "tempfile",
 ]
 
@@ -5446,7 +5483,7 @@ dependencies = [
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -5500,15 +5537,16 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.14"
+version = "0.11.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
 dependencies = [
  "aws-lc-rs",
  "bytes",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.10.1",
+ "rand_pcg",
  "ring",
  "rustc-hash",
  "rustls",
@@ -5573,9 +5611,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -5643,7 +5681,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
+dependencies = [
+ "rand_core 0.10.1",
 ]
 
 [[package]]
@@ -5733,7 +5780,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6107,9 +6154,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -6299,7 +6346,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6366,9 +6413,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -6376,22 +6423,22 @@ dependencies = [
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -6402,15 +6449,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -6437,7 +6485,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6449,7 +6497,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6493,7 +6541,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6640,23 +6688,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522"
+checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7"
 dependencies = [
  "snafu-derive",
 ]
 
 [[package]]
 name = "snafu-derive"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda"
+checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6699,9 +6747,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -6715,7 +6763,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6786,16 +6834,17 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -6809,7 +6858,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn",
+ "syn 2.0.118",
  "typify",
  "walkdir",
 ]
@@ -6837,6 +6886,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "sync_wrapper"
 version = "1.0.2"
@@ -6854,7 +6914,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6943,7 +7003,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -6954,7 +7014,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -7043,9 +7103,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
 dependencies = [
  "bytes",
  "libc",
@@ -7066,7 +7126,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -7092,9 +7152,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -7104,13 +7164,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
+ "libc",
  "pin-project-lite",
  "tokio",
 ]
@@ -7288,7 +7349,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -7351,11 +7412,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "twox-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
 dependencies = [
- "rand 0.9.4",
+ "rand 0.10.1",
 ]
 
 [[package]]
@@ -7395,7 +7456,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn",
+ "syn 2.0.118",
  "thiserror 2.0.18",
  "unicode-ident",
 ]
@@ -7413,7 +7474,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn",
+ "syn 2.0.118",
  "typify-impl",
 ]
 
@@ -7500,9 +7561,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.4"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
  "getrandom 0.4.3",
  "js-sys",
@@ -7622,7 +7683,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
  "wasm-bindgen-shared",
 ]
 
@@ -7796,7 +7857,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -7807,7 +7868,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -8296,9 +8357,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.16"
+version = "0.8.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
+checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
 
 [[package]]
 name = "yoke"
@@ -8319,7 +8380,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
  "synstructure",
 ]
 
@@ -8340,7 +8401,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -8360,7 +8421,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
  "synstructure",
 ]
 
@@ -8402,7 +8463,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml
index d8f839b273a..8dd254ef79a 100644
--- a/java/lance-jni/Cargo.toml
+++ b/java/lance-jni/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lance-jni"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 rust-version = "1.91"
@@ -20,6 +20,7 @@ crate-type = ["cdylib"]
 
 [features]
 default = []
+backtrace = ["lance/backtrace", "lance-core/backtrace"]
 
 [dependencies]
 lance = { path = "../../rust/lance", features = ["substrait"] }
@@ -36,8 +37,8 @@ lance-table = { path = "../../rust/lance-table" }
 arrow = { version = "58.0.0", features = ["ffi"] }
 arrow-array = "58.0.0"
 arrow-schema = "58.0.0"
-datafusion = { version = "53.0.0", default-features = false }
-datafusion-common = "53.0.0"
+datafusion = { version = "54.0.0", default-features = false }
+datafusion-common = "54.0.0"
 object_store = { version = "0.13.2" }
 tokio = { version = "1.23", features = [
     "rt-multi-thread",
diff --git a/java/lance-jni/src/blocking_blob.rs b/java/lance-jni/src/blocking_blob.rs
index 002fa817cf6..d75596d17d9 100755
--- a/java/lance-jni/src/blocking_blob.rs
+++ b/java/lance-jni/src/blocking_blob.rs
@@ -63,19 +63,22 @@ fn inner_take_blobs<'local>(
     };
     let j_blobs = blobs
         .into_iter()
-        .map(BlockingBlobFile::from)
-        .collect::<Vec<BlockingBlobFile>>();
+        .map(|blob| blob.map(BlockingBlobFile::from))
+        .collect::<Vec<_>>();
     transform_vec(env, j_blobs)
 }
 
 fn transform_vec<'local>(
     env: &mut JNIEnv<'local>,
-    vec: Vec<BlockingBlobFile>,
+    vec: Vec<Option<BlockingBlobFile>>,
 ) -> Result<JObject<'local>> {
     let array_list_class = env.find_class("java/util/ArrayList")?;
     let array_list = env.new_object(array_list_class, "()V", &[])?;
     for blob_file in vec {
-        let blob_file_obj = blob_file.into_java(env)?;
+        let blob_file_obj = match blob_file {
+            Some(blob_file) => blob_file.into_java(env)?,
+            None => JObject::null(),
+        };
         env.call_method(
             &array_list,
             "add",
@@ -117,8 +120,8 @@ fn inner_take_blobs_by_indices<'local>(
     };
     let j_blobs = blobs
         .into_iter()
-        .map(BlockingBlobFile::from)
-        .collect::<Vec<BlockingBlobFile>>();
+        .map(|blob| blob.map(BlockingBlobFile::from))
+        .collect::<Vec<_>>();
     transform_vec(env, j_blobs)
 }
 
diff --git a/java/lance-jni/src/blocking_dataset.rs b/java/lance-jni/src/blocking_dataset.rs
index caf837b371a..767f11de4df 100644
--- a/java/lance-jni/src/blocking_dataset.rs
+++ b/java/lance-jni/src/blocking_dataset.rs
@@ -1228,8 +1228,10 @@ fn index_metadata_to_segment(metadata: &IndexMetadata) -> Result<IndexSegment> {
     Ok(IndexSegment::new(
         metadata.uuid,
         fragment_bitmap,
+        metadata.fields.iter().copied(),
         index_details,
         metadata.index_version,
+        metadata.dataset_version,
     ))
 }
 
diff --git a/java/lance-jni/src/error.rs b/java/lance-jni/src/error.rs
index cdb922a3cef..0affdca8f98 100644
--- a/java/lance-jni/src/error.rs
+++ b/java/lance-jni/src/error.rs
@@ -181,29 +181,36 @@ impl std::fmt::Display for Error {
 
 impl From<LanceError> for Error {
     fn from(err: LanceError) -> Self {
+        let backtrace_suffix = err
+            .backtrace()
+            .map(|bt| format!("\n\nRust backtrace:\n{}", bt))
+            .unwrap_or_default();
+        let message = format!("{}{}", err, backtrace_suffix);
+
         match &err {
             LanceError::DatasetNotFound { .. }
             | LanceError::DatasetAlreadyExists { .. }
             | LanceError::CommitConflict { .. }
-            | LanceError::InvalidInput { .. } => Self::input_error(err.to_string()),
-            LanceError::IO { .. } => Self::io_error(err.to_string()),
-            LanceError::Timeout { .. } => Self::timeout_error(err.to_string()),
-            LanceError::NotSupported { .. } => Self::unsupported_error(err.to_string()),
-            LanceError::NotFound { .. } => Self::io_error(err.to_string()),
+            | LanceError::InvalidInput { .. } => Self::input_error(message),
+            LanceError::IO { .. } => Self::io_error(message),
+            LanceError::Timeout { .. } => Self::timeout_error(message),
+            LanceError::NotSupported { .. } => Self::unsupported_error(message),
+            LanceError::NotFound { .. } => Self::io_error(message),
             LanceError::Namespace { source, .. } => {
                 // Try to downcast to NamespaceError and get the error code
                 if let Some(ns_err) = source.downcast_ref::<NamespaceError>() {
-                    Self::namespace_error(ns_err.code().as_u32(), ns_err.to_string())
+                    let ns_message = format!("{}{}", ns_err, backtrace_suffix);
+                    Self::namespace_error(ns_err.code().as_u32(), ns_message)
                 } else {
                     log::warn!(
                         "Failed to downcast NamespaceError source, falling back to runtime error. \
                          This may indicate a version mismatch. Source type: {:?}",
                         source
                     );
-                    Self::runtime_error(err.to_string())
+                    Self::runtime_error(message)
                 }
             }
-            _ => Self::runtime_error(err.to_string()),
+            _ => Self::runtime_error(message),
         }
     }
 }
@@ -241,3 +248,104 @@ impl From<Utf8Error> for Error {
         Self::input_error(err.to_string())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Helper: extract the java_class from an Error via Display output
+    fn java_class(err: &Error) -> &JavaExceptionClass {
+        &err.java_class
+    }
+
+    #[test]
+    fn test_invalid_input_maps_to_illegal_argument() {
+        let lance_err = LanceError::invalid_input("bad input");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("bad input"));
+    }
+
+    #[test]
+    fn test_dataset_not_found_maps_to_illegal_argument() {
+        let lance_err = LanceError::dataset_not_found("my_dataset", "not found".to_string().into());
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("my_dataset"));
+    }
+
+    #[test]
+    fn test_dataset_already_exists_maps_to_illegal_argument() {
+        let lance_err = LanceError::dataset_already_exists("my_dataset");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("my_dataset"));
+    }
+
+    #[test]
+    fn test_commit_conflict_maps_to_illegal_argument() {
+        let lance_err = LanceError::commit_conflict_source(42, "conflict".to_string().into());
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+    }
+
+    #[test]
+    fn test_io_maps_to_ioexception() {
+        let lance_err = LanceError::io("disk failure");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException);
+        assert!(jni_err.message.contains("disk failure"));
+    }
+
+    #[test]
+    fn test_not_supported_maps_to_unsupported() {
+        let lance_err = LanceError::not_supported("nope");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::UnsupportedOperationException
+        );
+        assert!(jni_err.message.contains("nope"));
+    }
+
+    #[test]
+    fn test_not_found_maps_to_ioexception() {
+        let lance_err = LanceError::not_found("missing_uri");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException);
+        assert!(jni_err.message.contains("missing_uri"));
+    }
+
+    #[test]
+    fn test_fallthrough_maps_to_runtime() {
+        let lance_err = LanceError::internal("internal oops");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::RuntimeException);
+        assert!(jni_err.message.contains("internal oops"));
+    }
+
+    #[test]
+    fn test_no_backtrace_suffix_when_backtrace_is_none() {
+        // Without the backtrace feature enabled in lance-core default tests,
+        // backtrace() returns None, so no suffix should be appended.
+        let lance_err = LanceError::io("clean message");
+        let jni_err: Error = lance_err.into();
+        assert!(
+            !jni_err.message.contains("Rust backtrace:"),
+            "Expected no backtrace suffix, got: {}",
+            jni_err.message
+        );
+    }
+}
diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs
index d6603925947..59ce5e553ff 100644
--- a/java/lance-jni/src/fragment.rs
+++ b/java/lance-jni/src/fragment.rs
@@ -828,6 +828,9 @@ impl FromJObjectWithEnv<Fragment> for JObject<'_> {
             row_id_meta,
             created_at_version_meta,
             last_updated_at_version_meta,
+            // Overlays are not exposed to Java yet, and the reverse conversion
+            // does not export them, so this round-trip is overlay-free.
+            overlays: vec![],
         })
     }
 }
diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs
index 37fe377ed17..3c11ee8da25 100644
--- a/java/lance-jni/src/mem_wal.rs
+++ b/java/lance-jni/src/mem_wal.rs
@@ -26,7 +26,7 @@ use jni::objects::{JClass, JMap, JObject, JString, JValueGen};
 use jni::sys::{jdouble, jint, jlong};
 use lance::dataset::Dataset as LanceDataset;
 use lance::dataset::mem_wal::scanner::{
-    FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner,
+    LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, SsTable,
     parse_filter_expr as parse_lsm_filter_expr, write_pk_sidecar,
 };
 use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot};
@@ -181,10 +181,33 @@ fn inner_put(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()>
     Ok(())
 }
 
-/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for a
-/// flushed-generation dataset already staged at `gen_path`, mirroring what
-/// production flush emits. Lets Java tests stage a *faithful* flushed
-/// generation (dataset + sidecar); production always writes the sidecar during
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_memwal_ShardWriter_nativeDelete(
+    mut env: JNIEnv,
+    this: JObject,
+    stream_addr: jlong,
+) {
+    ok_or_throw_without_return!(env, inner_delete(&mut env, this, stream_addr));
+}
+
+fn inner_delete(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> {
+    let stream_ptr = stream_addr as *mut FFI_ArrowArrayStream;
+    let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
+    let batches: Vec<RecordBatch> = reader.collect::<std::result::Result<_, _>>()?;
+    if batches.is_empty() {
+        return Ok(());
+    }
+
+    let guard =
+        unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?;
+    RT.block_on(guard.writer.delete(batches))?;
+    Ok(())
+}
+
+/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for an
+/// SSTable dataset already staged at `gen_path`, mirroring what
+/// production flush emits. Lets Java tests stage a *faithful* SSTable
+/// (dataset + sidecar); production always writes the sidecar during
 /// flush, so a dataset-without-sidecar is not a state the system produces.
 /// Mirrors the Python `_write_pk_sidecar` binding.
 #[unsafe(no_mangle)]
@@ -1096,13 +1119,13 @@ fn read_shard_snapshots(env: &mut JNIEnv, list_obj: &JObject) -> Result<Vec<Shar
             .with_spec_id(spec_id)
             .with_current_generation(current_generation);
 
-        let flushed_list = env
-            .call_method(&obj, "flushedGenerations", "()Ljava/util/List;", &[])?
+        let sstable_list = env
+            .call_method(&obj, "sstables", "()Ljava/util/List;", &[])?
             .l()?;
-        for flushed in import_vec(env, &flushed_list)? {
-            let generation = env.get_u64_from_method(&flushed, "generation")?;
-            let path = env.get_string_from_method(&flushed, "path")?;
-            snapshot = snapshot.with_flushed_generation(generation, path);
+        for sstable in import_vec(env, &sstable_list)? {
+            let generation = env.get_u64_from_method(&sstable, "generation")?;
+            let path = env.get_string_from_method(&sstable, "path")?;
+            snapshot = snapshot.with_sstable(generation, path);
         }
         Ok(snapshot)
     })
@@ -1114,12 +1137,12 @@ fn shard_snapshot_from_manifest(manifest: ShardManifest) -> ShardSnapshot {
         shard_id: manifest.shard_id,
         spec_id: manifest.shard_spec_id,
         current_generation: manifest.current_generation,
-        flushed_generations: manifest
-            .flushed_generations
+        sstables: manifest
+            .sstables
             .into_iter()
-            .map(|generation| FlushedGeneration {
-                generation: generation.generation,
-                path: generation.path,
+            .map(|sstable| SsTable {
+                generation: sstable.generation,
+                path: sstable.path,
             })
             .collect(),
     }
@@ -1242,9 +1265,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result<ShardWriter
     if let Some(v) = read_optional_bool(env, config, "durableWrite")? {
         writer_config = writer_config.with_durable_write(v);
     }
-    if let Some(v) = read_optional_bool(env, config, "syncIndexedWrite")? {
-        writer_config = writer_config.with_sync_indexed_write(v);
-    }
     if let Some(v) = read_optional_u64(env, config, "maxWalBufferSize")? {
         writer_config = writer_config.with_max_wal_buffer_size(v as usize);
     }
@@ -1266,12 +1286,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result<ShardWriter
     if let Some(v) = read_optional_u64(env, config, "manifestScanBatchSize")? {
         writer_config = writer_config.with_manifest_scan_batch_size(v as usize);
     }
-    if let Some(v) = read_optional_u64(env, config, "asyncIndexBufferRows")? {
-        writer_config = writer_config.with_async_index_buffer_rows(v as usize);
-    }
-    if let Some(v) = read_optional_u64(env, config, "asyncIndexIntervalMs")? {
-        writer_config = writer_config.with_async_index_interval(Duration::from_millis(v));
-    }
     if let Some(v) = read_optional_u64(env, config, "backpressureLogIntervalMs")? {
         writer_config = writer_config.with_backpressure_log_interval(Duration::from_millis(v));
     }
@@ -1345,19 +1359,19 @@ fn write_stats_to_java<'a>(
 
 fn memtable_stats_to_java<'a>(env: &mut JNIEnv<'a>, stats: &MemTableStats) -> Result<JObject<'a>> {
     let max_buffered = box_u64_opt(env, stats.max_buffered_batch_position)?;
-    let max_flushed = box_u64_opt(env, stats.max_flushed_batch_position)?;
     let pending_start = box_u64_opt(env, stats.pending_wal_start_batch_position)?;
     let pending_end = box_u64_opt(env, stats.pending_wal_end_batch_position)?;
     Ok(env.new_object(
         "org/lance/memwal/MemTableStats",
-        "(JJJJLjava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;JJJ)V",
+        "(JJJJLjava/lang/Long;JJLjava/lang/Long;Ljava/lang/Long;JJJ)V",
         &[
             JValueGen::Long(stats.row_count as i64),
             JValueGen::Long(stats.batch_count as i64),
             JValueGen::Long(stats.estimated_size as i64),
             JValueGen::Long(stats.generation as i64),
             JValueGen::Object(&max_buffered),
-            JValueGen::Object(&max_flushed),
+            JValueGen::Long(stats.durable_batch_count as i64),
+            JValueGen::Long(stats.global_offset as i64),
             JValueGen::Object(&pending_start),
             JValueGen::Object(&pending_end),
             JValueGen::Long(stats.pending_wal_batch_count as i64),
diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs
index df4d63bd2f6..a7cedaea049 100644
--- a/java/lance-jni/src/merge_insert.rs
+++ b/java/lance-jni/src/merge_insert.rs
@@ -15,7 +15,7 @@ use lance::dataset::{
     MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource,
 };
 use lance_core::datatypes::Schema;
-use lance_index::mem_wal::MergedGeneration;
+use lance_index::mem_wal::CompactedSsTable;
 use std::sync::Arc;
 use std::time::Duration;
 use uuid::Uuid;
@@ -52,7 +52,7 @@ fn inner_merge_insert<'local>(
     let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?;
     let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?;
     let use_index = extract_use_index(env, &jparam)?;
-    let marked_generations = extract_marked_generations(env, &jparam)?;
+    let compacted_sstables = extract_compacted_sstables(env, &jparam)?;
 
     let (new_ds, merge_stats) = unsafe {
         let dataset = env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET)?;
@@ -71,7 +71,7 @@ fn inner_merge_insert<'local>(
             .retry_timeout(Duration::from_millis(retry_timeout_ms as u64))
             .skip_auto_cleanup(skip_auto_cleanup)
             .use_index(use_index)
-            .mark_generations_as_merged(marked_generations)
+            .mark_sstables_as_compacted(compacted_sstables)
             .try_build()?;
 
         let stream_ptr = batch_address as *mut FFI_ArrowArrayStream;
@@ -241,23 +241,23 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu
     Ok(use_index)
 }
 
-fn extract_marked_generations<'local>(
+fn extract_compacted_sstables<'local>(
     env: &mut JNIEnv<'local>,
     jparam: &JObject,
-) -> Result<Vec<MergedGeneration>> {
+) -> Result<Vec<CompactedSsTable>> {
     let list = env
-        .call_method(jparam, "markedGenerations", "()Ljava/util/List;", &[])?
+        .call_method(jparam, "getCompactedSstables", "()Ljava/util/List;", &[])?
         .l()?;
     import_vec_to_rust(env, &list, |env, obj| {
         let shard_id: JString = env
-            .call_method(&obj, "shardId", "()Ljava/lang/String;", &[])?
+            .call_method(&obj, "getShardId", "()Ljava/lang/String;", &[])?
             .l()?
             .into();
         let shard_id = shard_id.extract(env)?;
-        let generation = env.call_method(&obj, "generation", "()J", &[])?.j()? as u64;
+        let generation = env.call_method(&obj, "getGeneration", "()J", &[])?.j()? as u64;
         let uuid = Uuid::parse_str(&shard_id)
             .map_err(|e| Error::input_error(format!("Invalid shard_id UUID: {}", e)))?;
-        Ok(MergedGeneration::new(uuid, generation))
+        Ok(CompactedSsTable::new(uuid, generation))
     })
 }
 
diff --git a/java/lance-jni/src/session.rs b/java/lance-jni/src/session.rs
index 17b59c12ecb..8da6251451f 100644
--- a/java/lance-jni/src/session.rs
+++ b/java/lance-jni/src/session.rs
@@ -4,12 +4,13 @@
 use std::sync::Arc;
 
 use jni::JNIEnv;
-use jni::objects::JObject;
+use jni::objects::{JObject, JValue};
 use jni::sys::jlong;
 use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE};
 use lance::session::Session as LanceSession;
 use lance_io::object_store::ObjectStoreRegistry;
 
+use crate::RT;
 use crate::error::{Error, Result};
 
 /// Creates a new Session and returns a handle to it.
@@ -78,6 +79,41 @@ fn size_bytes_native(env: &mut JNIEnv, obj: JObject) -> Result<jlong> {
     Ok(session_arc.size_bytes() as jlong)
 }
 
+/// Returns statistics for the session's metadata cache as an org.lance.CacheStats object.
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Session_metadataCacheStatsNative<'local>(
+    mut env: JNIEnv<'local>,
+    obj: JObject,
+) -> JObject<'local> {
+    ok_or_throw!(env, metadata_cache_stats_native(&mut env, obj))
+}
+
+fn metadata_cache_stats_native<'local>(
+    env: &mut JNIEnv<'local>,
+    obj: JObject,
+) -> Result<JObject<'local>> {
+    let handle = get_session_handle(env, &obj)?;
+    if handle == 0 {
+        return Err(Error::input_error("Session is closed".to_string()));
+    }
+
+    // Safety: We trust that the handle is valid and was created by createNative
+    let session_arc = unsafe { &*(handle as *const Arc<LanceSession>) };
+    let stats = RT.block_on(session_arc.metadata_cache_stats());
+
+    let stats_obj = env.new_object(
+        "org/lance/CacheStats",
+        "(JJJJ)V",
+        &[
+            JValue::Long(stats.hits as jlong),
+            JValue::Long(stats.misses as jlong),
+            JValue::Long(stats.num_entries as jlong),
+            JValue::Long(stats.size_bytes as jlong),
+        ],
+    )?;
+    Ok(stats_obj)
+}
+
 /// Releases the native session handle.
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_Session_releaseNative(
diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs
index 4f899f56ff2..fba2358d337 100644
--- a/java/lance-jni/src/transaction.rs
+++ b/java/lance-jni/src/transaction.rs
@@ -429,7 +429,7 @@ fn convert_to_java_operation_inner<'local>(
             updated_fragments,
             new_fragments,
             fields_modified,
-            merged_generations: _,
+            compacted_sstables: _,
             fields_for_preserving_frag_bitmap,
             update_mode,
             inserted_rows_filter: _,
@@ -1243,7 +1243,7 @@ fn convert_to_rust_operation(
                 updated_fragments,
                 new_fragments,
                 fields_modified,
-                merged_generations: vec![],
+                compacted_sstables: vec![],
                 fields_for_preserving_frag_bitmap,
                 update_mode,
                 inserted_rows_filter: None,
diff --git a/java/pom.xml b/java/pom.xml
index 89b567c5f3e..04c71a4e70c 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -7,7 +7,7 @@
     <groupId>org.lance</groupId>
     <artifactId>lance-core</artifactId>
     <name>Lance Core</name>
-    <version>9.0.0-beta.16</version>
+    <version>10.0.0-beta.3</version>
     <packaging>jar</packaging>
 
     <description>Lance Format Java API</description>
@@ -39,6 +39,7 @@
         <spotless.scala.scalafmt.version>3.7.5</spotless.scala.scalafmt.version>
         <spotless.delimiter>package</spotless.delimiter>
         <rust.release.build>false</rust.release.build>
+        <rust.features></rust.features>
         <skip.build.jni>false</skip.build.jni>
         <shade.base>org.lance.shaded</shade.base>
         <spotless.license.header>
@@ -396,6 +397,9 @@
                                 <configuration>
                                     <path>lance-jni</path>
                                     <release>${rust.release.build}</release>
+                                    <features>
+                                        <feature>${rust.features}</feature>
+                                    </features>
                                     <!-- Copy native libraries to target/classes for runtime access -->
                                     <copyTo>${project.build.directory}/classes/nativelib</copyTo>
                                     <copyWithPlatformDir>true</copyWithPlatformDir>
@@ -409,6 +413,9 @@
                                 <configuration>
                                     <path>lance-jni</path>
                                     <release>${rust.release.build}</release>
+                                    <features>
+                                        <feature>${rust.features}</feature>
+                                    </features>
                                     <verbosity>-v</verbosity>
                                 </configuration>
                             </execution>
diff --git a/java/src/main/java/org/lance/CacheStats.java b/java/src/main/java/org/lance/CacheStats.java
new file mode 100644
index 00000000000..9df3388dabb
--- /dev/null
+++ b/java/src/main/java/org/lance/CacheStats.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+/**
+ * Statistics for a session cache.
+ *
+ * <p>A snapshot of cache activity, useful for monitoring hit/miss rates over time.
+ */
+public class CacheStats {
+  private final long hits;
+  private final long misses;
+  private final long numEntries;
+  private final long sizeBytes;
+
+  /**
+   * Constructs cache statistics. Instances are created by the native layer.
+   *
+   * @param hits number of cache lookups that found an item
+   * @param misses number of cache lookups that did not find an item
+   * @param numEntries number of entries currently in the cache
+   * @param sizeBytes total size in bytes of all entries in the cache
+   */
+  public CacheStats(long hits, long misses, long numEntries, long sizeBytes) {
+    if (hits < 0 || misses < 0 || numEntries < 0 || sizeBytes < 0) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Cache statistics must be non-negative: "
+                  + "hits=%d, misses=%d, numEntries=%d, sizeBytes=%d",
+              hits, misses, numEntries, sizeBytes));
+    }
+    this.hits = hits;
+    this.misses = misses;
+    this.numEntries = numEntries;
+    this.sizeBytes = sizeBytes;
+  }
+
+  /**
+   * Returns the number of cache lookups that found an item in the cache.
+   *
+   * @return the number of cache hits
+   */
+  public long getHits() {
+    return hits;
+  }
+
+  /**
+   * Returns the number of cache lookups that did not find an item in the cache.
+   *
+   * @return the number of cache misses
+   */
+  public long getMisses() {
+    return misses;
+  }
+
+  /**
+   * Returns the number of entries currently in the cache.
+   *
+   * @return the number of cache entries
+   */
+  public long getNumEntries() {
+    return numEntries;
+  }
+
+  /**
+   * Returns the total size in bytes of all entries in the cache.
+   *
+   * @return the cache size in bytes
+   */
+  public long getSizeBytes() {
+    return sizeBytes;
+  }
+
+  /**
+   * Returns the ratio of hits to total lookups, or 0 if there have been no lookups.
+   *
+   * @return the cache hit ratio in the range [0, 1]
+   */
+  public double getHitRatio() {
+    // Sum in double to avoid long overflow for very large counters
+    double total = (double) hits + (double) misses;
+    if (total == 0) {
+      return 0.0;
+    }
+    return hits / total;
+  }
+
+  @Override
+  public String toString() {
+    return String.format(
+        "CacheStats(hits=%d, misses=%d, numEntries=%d, sizeBytes=%d)",
+        hits, misses, numEntries, sizeBytes);
+  }
+}
diff --git a/java/src/main/java/org/lance/Dataset.java b/java/src/main/java/org/lance/Dataset.java
index 8f79d7cbaea..176089ca4e0 100644
--- a/java/src/main/java/org/lance/Dataset.java
+++ b/java/src/main/java/org/lance/Dataset.java
@@ -1621,11 +1621,27 @@ public void close() {
   private native List<BlobFile> nativeTakeBlobsByIndices(List<Long> rowIndices, String column);
 
   /**
-   * Open blob files for given row ids on a blob column. Names and semantics align with Rust/Python.
+   * Open {@link BlobFile} handles for given row IDs on a blob column. Names and semantics align
+   * with Rust/Python.
    *
-   * @param rowIds stable row ids (row addresses)
+   * <p>Pass logical row IDs read from {@code _rowid}, not physical row addresses from {@code
+   * _rowaddr}.
+   *
+   * <pre>{@code
+   * long rowId = 42L; // Example value from the _rowid column.
+   * List<BlobFile> blobs = dataset.takeBlobs(List.of(rowId), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }</pre>
+   *
+   * @param rowIds logical row IDs from the {@code _rowid} column
    * @param column blob column name
-   * @return list of BlobFile objects
+   * @return one {@link BlobFile} per row ID; null blob values are represented by null elements
    */
   public List<BlobFile> takeBlobs(List<Long> rowIds, String column) {
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
@@ -1639,11 +1655,22 @@ public List<BlobFile> takeBlobs(List<Long> rowIds, String column) {
   }
 
   /**
-   * Open blob files for given row indices on a blob column.
+   * Open {@link BlobFile} handles for given row indices on a blob column.
+   *
+   * <pre>{@code
+   * List<BlobFile> blobs = dataset.takeBlobsByIndices(List.of(0L), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }</pre>
    *
    * @param rowIndices row offsets within dataset
    * @param column blob column name
-   * @return list of BlobFile objects
+   * @return one {@link BlobFile} per row index; null blob values are represented by null elements
    */
   public List<BlobFile> takeBlobsByIndices(List<Long> rowIndices, String column) {
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
diff --git a/java/src/main/java/org/lance/Session.java b/java/src/main/java/org/lance/Session.java
index 0fe4a59736c..04ee1bb4984 100644
--- a/java/src/main/java/org/lance/Session.java
+++ b/java/src/main/java/org/lance/Session.java
@@ -176,6 +176,19 @@ public long sizeBytes() {
     return sizeBytesNative();
   }
 
+  /**
+   * Returns statistics for the metadata cache of this session.
+   *
+   * <p>The returned statistics are a snapshot; call this method periodically to observe how hits
+   * and misses evolve over time.
+   *
+   * @return the metadata cache statistics
+   */
+  public CacheStats metadataCacheStats() {
+    Preconditions.checkArgument(nativeSessionHandle != 0, "Session is closed");
+    return metadataCacheStatsNative();
+  }
+
   /**
    * Returns whether the other session is the same as this one.
    *
@@ -242,6 +255,8 @@ public String toString() {
 
   private native long sizeBytesNative();
 
+  private native CacheStats metadataCacheStatsNative();
+
   private static native void releaseNative(long handle);
 
   private static native boolean isSameAsNative(long handle1, long handle2);
diff --git a/java/src/main/java/org/lance/WriteDatasetBuilder.java b/java/src/main/java/org/lance/WriteDatasetBuilder.java
index 96a894fc944..9d406406291 100644
--- a/java/src/main/java/org/lance/WriteDatasetBuilder.java
+++ b/java/src/main/java/org/lance/WriteDatasetBuilder.java
@@ -70,6 +70,7 @@ public class WriteDatasetBuilder {
   private WriteParams.WriteMode mode = WriteParams.WriteMode.CREATE;
   private Schema schema;
   private Map<String, String> storageOptions = new HashMap<>();
+  private Map<String, String> properties = new HashMap<>();
   private Map<String, Map<String, String>> baseStoreParams = new HashMap<>();
   private boolean ignoreNamespaceStorageOptions = false;
   private Optional<Integer> maxRowsPerFile = Optional.empty();
@@ -207,6 +208,27 @@ public WriteDatasetBuilder storageOptions(Map<String, String> storageOptions) {
     return this;
   }
 
+  /**
+   * Sets the table properties to forward to the namespace on table creation.
+   *
+   * <p>These are Lance-namespace <b>properties</b>: catalog-level key-value metadata stored by the
+   * namespace outside the Lance table (available even if the table manifest does not exist), as
+   * distinct from the manifest-stored {@code config} (read/write behavior) and {@code metadata}
+   * (business metadata), and from non-persisted {@code storageOptions}. They are attached to the
+   * underlying declareTable request via its {@code properties} field.
+   *
+   * <p>Only used when a namespace client is configured via namespaceClient()+tableId() and the
+   * write creates the table (CREATE mode). Ignored for direct-URI writes and for APPEND/OVERWRITE
+   * modes.
+   *
+   * @param properties Table properties to forward on declareTable
+   * @return this builder instance
+   */
+  public WriteDatasetBuilder properties(Map<String, String> properties) {
+    this.properties = new HashMap<>(properties);
+    return this;
+  }
+
   /**
    * Sets runtime-only object store parameters for registered base paths.
    *
@@ -410,6 +432,9 @@ private Dataset executeWithNamespaceClient() {
     if (mode == WriteParams.WriteMode.CREATE) {
       DeclareTableRequest declareRequest = new DeclareTableRequest();
       declareRequest.setId(tableId);
+      if (properties != null && !properties.isEmpty()) {
+        declareRequest.setProperties(properties);
+      }
       DeclareTableResponse declareResponse = namespaceClient.declareTable(declareRequest);
 
       tableUri = declareResponse.getLocation();
diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
index 9b29d7a0795..ac23fd0d900 100755
--- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
+++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
@@ -53,6 +53,7 @@ public static final class Builder {
     private Integer minNgramLength;
     private Integer maxNgramLength;
     private Boolean prefixOnly;
+    private Integer blockSize = 128;
     private Boolean skipMerge;
     private Integer formatVersion;
 
@@ -66,6 +67,7 @@ public static final class Builder {
      *   <li>{@code "whitespace"}: splits tokens on whitespace
      *   <li>{@code "raw"}: no tokenization
      *   <li>{@code "ngram"}: N-Gram tokenizer
+     *   <li>{@code "code"}: code-aware tokenizer
      *   <li>{@code "icu"}: ICU dictionary-based Unicode word segmentation
      *   <li>{@code "icu/split"}: ICU segmentation with simple-style delimiter splitting
      *   <li>{@code "lindera/*"}: Lindera tokenizer
@@ -226,6 +228,27 @@ public Builder prefixOnly(boolean prefixOnly) {
       return this;
     }
 
+    /**
+     * Configure the number of documents in each compressed posting block.
+     *
+     * <p>Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when
+     * this is not set.
+     *
+     * <p>{@code blockSize = 256} requires FTS format v3. Format v3 also supports the default {@code
+     * blockSize = 128}.
+     *
+     * @param blockSize posting block size
+     * @return this builder
+     * @throws IllegalArgumentException if {@code blockSize} is unsupported
+     */
+    public Builder blockSize(int blockSize) {
+      if (blockSize != 128 && blockSize != 256) {
+        throw new IllegalArgumentException("blockSize must be one of 128 or 256");
+      }
+      this.blockSize = blockSize;
+      return this;
+    }
+
     /**
      * Configure whether to skip the partition merge stage after indexing. If true, skip the
      * partition merge stage after indexing. This can be useful for distributed indexing where merge
@@ -242,15 +265,18 @@ public Builder skipMerge(boolean skipMerge) {
     /**
      * Configure the on-disk FTS format version to write when creating a new index.
      *
-     * <p>If unset, Lance chooses the current default format.
+     * <p>If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects
+     * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3
+     * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and
+     * cannot be used with the code analyzer.
      *
-     * @param formatVersion FTS format version, must be 1 or 2
+     * @param formatVersion FTS format version, must be 1, 2, or 3
      * @return this builder
      * @throws IllegalArgumentException
      */
     public Builder formatVersion(int formatVersion) {
-      if (formatVersion != 1 && formatVersion != 2) {
-        throw new IllegalArgumentException("formatVersion must be 1 or 2");
+      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) {
+        throw new IllegalArgumentException("formatVersion must be 1, 2, or 3");
       }
       this.formatVersion = formatVersion;
       return this;
@@ -258,6 +284,13 @@ public Builder formatVersion(int formatVersion) {
 
     /** Build a {@link ScalarIndexParams} instance for an inverted index. */
     public ScalarIndexParams build() {
+      if (formatVersion != null) {
+        Preconditions.checkArgument(
+            formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128");
+        Preconditions.checkArgument(
+            !"code".equals(baseTokenizer) || formatVersion == 3,
+            "baseTokenizer 'code' requires formatVersion 3");
+      }
       Map<String, Object> params = new HashMap<>();
       if (baseTokenizer != null) {
         params.put("base_tokenizer", baseTokenizer);
@@ -300,6 +333,9 @@ public ScalarIndexParams build() {
       if (prefixOnly != null) {
         params.put("prefix_only", prefixOnly);
       }
+      if (blockSize != null) {
+        params.put("block_size", blockSize);
+      }
       if (skipMerge != null) {
         params.put("skip_merge", skipMerge);
       }
diff --git a/java/src/main/java/org/lance/ipc/ScanOptions.java b/java/src/main/java/org/lance/ipc/ScanOptions.java
index a9aad590c2b..7e1b4222262 100644
--- a/java/src/main/java/org/lance/ipc/ScanOptions.java
+++ b/java/src/main/java/org/lance/ipc/ScanOptions.java
@@ -134,6 +134,8 @@ public ScanOptions(
     Preconditions.checkArgument(
         !(filter.isPresent() && substraitFilter.isPresent()),
         "cannot set both substrait filter and string filter");
+    Preconditions.checkArgument(
+        batchReadahead > 0, "batchReadahead must be greater than 0, got %s", batchReadahead);
     this.fragmentIds = fragmentIds;
     this.batchSize = batchSize;
     this.columns = columns;
diff --git a/java/src/main/java/org/lance/memwal/MergedGeneration.java b/java/src/main/java/org/lance/memwal/CompactedSsTable.java
similarity index 77%
rename from java/src/main/java/org/lance/memwal/MergedGeneration.java
rename to java/src/main/java/org/lance/memwal/CompactedSsTable.java
index 481e79ba3c0..cb703b42641 100644
--- a/java/src/main/java/org/lance/memwal/MergedGeneration.java
+++ b/java/src/main/java/org/lance/memwal/CompactedSsTable.java
@@ -17,32 +17,32 @@
 import com.google.common.base.Preconditions;
 
 /**
- * Identifies a flushed MemWAL generation that has been merged into the base table.
+ * Points to an SSTable compacted into the base table.
  *
- * <p>Pass a list of these to {@link org.lance.merge.MergeInsertParams#markGenerationsAsMerged} so
- * Lance knows which generations are now part of the base table.
+ * <p>Pass a list of these to {@link org.lance.merge.MergeInsertParams#markSstablesAsCompacted} so
+ * Lance can record compaction progress.
  */
-public class MergedGeneration {
+public class CompactedSsTable {
   private final String shardId;
   private final long generation;
 
   /**
    * @param shardId UUID string for the write shard
-   * @param generation generation number from {@link ShardSnapshot#flushedGenerations()}
+   * @param generation generation number from {@link ShardSnapshot#sstables()}
    */
-  public MergedGeneration(String shardId, long generation) {
+  public CompactedSsTable(String shardId, long generation) {
     Preconditions.checkNotNull(shardId, "shardId must not be null");
     this.shardId = shardId;
     this.generation = generation;
   }
 
   /** UUID string for the write shard. */
-  public String shardId() {
+  public String getShardId() {
     return shardId;
   }
 
-  /** The merged generation number. */
-  public long generation() {
+  /** The compacted SSTable's generation number. */
+  public long getGeneration() {
     return generation;
   }
 
diff --git a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java
index 0488f9e4530..d58e4e81d57 100644
--- a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java
+++ b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java
@@ -44,7 +44,7 @@ public class LsmPointLookupPlanner implements AutoCloseable {
 
   /**
    * @param dataset the base dataset
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    */
   public LsmPointLookupPlanner(Dataset dataset, List<ShardSnapshot> shardSnapshots) {
     this(dataset, shardSnapshots, null);
@@ -52,7 +52,7 @@ public LsmPointLookupPlanner(Dataset dataset, List<ShardSnapshot> shardSnapshots
 
   /**
    * @param dataset the base dataset
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    * @param pkColumns primary key column names; inferred from schema metadata when {@code null}
    */
   public LsmPointLookupPlanner(
diff --git a/java/src/main/java/org/lance/memwal/LsmScanner.java b/java/src/main/java/org/lance/memwal/LsmScanner.java
index 29bfb097375..509e70f7b0a 100644
--- a/java/src/main/java/org/lance/memwal/LsmScanner.java
+++ b/java/src/main/java/org/lance/memwal/LsmScanner.java
@@ -31,8 +31,8 @@
  * LSM-aware scanner covering all MemWAL data levels.
  *
  * <p>Results are deduplicated by primary key, always returning the newest version of each row
- * across the base table, flushed MemTables, and (when created from a {@link ShardWriter}) the
- * active MemTable.
+ * across the base table, SSTables, and (when created from a {@link ShardWriter}) the active
+ * MemTable.
  *
  * <p>The builder methods ({@link #project}, {@link #filter}, {@link #limit}, {@link
  * #withRowAddress}, {@link #withMemtableGen}) mutate this scanner and return it for chaining.
@@ -55,7 +55,7 @@ private LsmScanner() {}
    * read-your-writes consistency.
    *
    * @param dataset the base dataset to scan
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    * @return an LSM scanner
    */
   public static LsmScanner fromSnapshots(Dataset dataset, List<ShardSnapshot> shardSnapshots) {
diff --git a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
index 75c0d86c475..b17bb8e37a0 100644
--- a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
+++ b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
@@ -44,7 +44,7 @@ public class LsmVectorSearchPlanner implements AutoCloseable {
 
   /**
    * @param dataset the base dataset
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    * @param vectorColumn name of the {@code FixedSizeList<float32>} vector column
    */
   public LsmVectorSearchPlanner(
@@ -54,7 +54,7 @@ public LsmVectorSearchPlanner(
 
   /**
    * @param dataset the base dataset
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    * @param vectorColumn name of the {@code FixedSizeList<float32>} vector column
    * @param pkColumns primary key column names; inferred from schema metadata when {@code null}
    * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"},
@@ -71,7 +71,7 @@ public LsmVectorSearchPlanner(
 
   /**
    * @param dataset the base dataset
-   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param shardSnapshots shard snapshots specifying the SSTables to include
    * @param vectorColumn name of the {@code FixedSizeList<float32>} vector column
    * @param pkColumns primary key column names; inferred from schema metadata when {@code null}
    * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"},
diff --git a/java/src/main/java/org/lance/memwal/MemTableStats.java b/java/src/main/java/org/lance/memwal/MemTableStats.java
index a993d771b42..b1fbeede299 100644
--- a/java/src/main/java/org/lance/memwal/MemTableStats.java
+++ b/java/src/main/java/org/lance/memwal/MemTableStats.java
@@ -24,7 +24,8 @@ public class MemTableStats {
   private final long estimatedSizeBytes;
   private final long generation;
   private final Optional<Long> maxBufferedBatchPosition;
-  private final Optional<Long> maxFlushedBatchPosition;
+  private final long durableBatchCount;
+  private final long globalOffset;
   private final Optional<Long> pendingWalStartBatchPosition;
   private final Optional<Long> pendingWalEndBatchPosition;
   private final long pendingWalBatchCount;
@@ -37,7 +38,8 @@ public MemTableStats(
       long estimatedSizeBytes,
       long generation,
       Long maxBufferedBatchPosition,
-      Long maxFlushedBatchPosition,
+      long durableBatchCount,
+      long globalOffset,
       Long pendingWalStartBatchPosition,
       Long pendingWalEndBatchPosition,
       long pendingWalBatchCount,
@@ -48,7 +50,8 @@ public MemTableStats(
     this.estimatedSizeBytes = estimatedSizeBytes;
     this.generation = generation;
     this.maxBufferedBatchPosition = Optional.ofNullable(maxBufferedBatchPosition);
-    this.maxFlushedBatchPosition = Optional.ofNullable(maxFlushedBatchPosition);
+    this.durableBatchCount = durableBatchCount;
+    this.globalOffset = globalOffset;
     this.pendingWalStartBatchPosition = Optional.ofNullable(pendingWalStartBatchPosition);
     this.pendingWalEndBatchPosition = Optional.ofNullable(pendingWalEndBatchPosition);
     this.pendingWalBatchCount = pendingWalBatchCount;
@@ -81,9 +84,17 @@ public Optional<Long> maxBufferedBatchPosition() {
     return maxBufferedBatchPosition;
   }
 
-  /** Highest WAL batch position flushed from the MemTable, if any. */
-  public Optional<Long> maxFlushedBatchPosition() {
-    return maxFlushedBatchPosition;
+  /**
+   * Writer-global count of WAL-durable batches (exclusive; 0 means none). Compare against {@code
+   * globalOffset() + batchCount()} to see what this MemTable still owes the WAL.
+   */
+  public long durableBatchCount() {
+    return durableBatchCount;
+  }
+
+  /** Writer-global coordinate of this MemTable's batch 0. */
+  public long globalOffset() {
+    return globalOffset;
   }
 
   /** First WAL batch position pending flush, if any. */
@@ -119,7 +130,8 @@ public String toString() {
         .add("estimatedSizeBytes", estimatedSizeBytes)
         .add("generation", generation)
         .add("maxBufferedBatchPosition", maxBufferedBatchPosition.orElse(null))
-        .add("maxFlushedBatchPosition", maxFlushedBatchPosition.orElse(null))
+        .add("durableBatchCount", durableBatchCount)
+        .add("globalOffset", globalOffset)
         .add("pendingWalStartBatchPosition", pendingWalStartBatchPosition.orElse(null))
         .add("pendingWalEndBatchPosition", pendingWalEndBatchPosition.orElse(null))
         .add("pendingWalBatchCount", pendingWalBatchCount)
diff --git a/java/src/main/java/org/lance/memwal/ShardSnapshot.java b/java/src/main/java/org/lance/memwal/ShardSnapshot.java
index 493b2fce496..42e20bb5177 100644
--- a/java/src/main/java/org/lance/memwal/ShardSnapshot.java
+++ b/java/src/main/java/org/lance/memwal/ShardSnapshot.java
@@ -24,13 +24,13 @@
  * Snapshot of a MemWAL shard's state, used when constructing scanners and planners.
  *
  * <p>The builder methods ({@link #withSpecId}, {@link #withCurrentGeneration}, {@link
- * #withFlushedGeneration}) mutate this instance and return it for chaining.
+ * #withSsTable}) mutate this instance and return it for chaining.
  */
 public class ShardSnapshot {
   private final String shardId;
   private int specId = 0;
   private long currentGeneration = 0;
-  private final List<FlushedGeneration> flushedGenerations = new ArrayList<>();
+  private final List<SsTable> sstables = new ArrayList<>();
 
   /**
    * @param shardId UUID string for the write shard
@@ -52,10 +52,10 @@ public ShardSnapshot withCurrentGeneration(long currentGeneration) {
     return this;
   }
 
-  /** Add a flushed generation with its storage path. */
-  public ShardSnapshot withFlushedGeneration(long generation, String path) {
+  /** Add an SSTable with its storage path. */
+  public ShardSnapshot withSsTable(long generation, String path) {
     Preconditions.checkNotNull(path, "path must not be null");
-    this.flushedGenerations.add(new FlushedGeneration(generation, path));
+    this.sstables.add(new SsTable(generation, path));
     return this;
   }
 
@@ -74,9 +74,9 @@ public long currentGeneration() {
     return currentGeneration;
   }
 
-  /** The flushed generations included in this snapshot. */
-  public List<FlushedGeneration> flushedGenerations() {
-    return Collections.unmodifiableList(flushedGenerations);
+  /** The SSTables included in this snapshot. */
+  public List<SsTable> sstables() {
+    return Collections.unmodifiableList(sstables);
   }
 
   @Override
@@ -85,7 +85,7 @@ public String toString() {
         .add("shardId", shardId)
         .add("specId", specId)
         .add("currentGeneration", currentGeneration)
-        .add("flushedGenerations", flushedGenerations)
+        .add("sstables", sstables)
         .toString();
   }
 }
diff --git a/java/src/main/java/org/lance/memwal/ShardWriter.java b/java/src/main/java/org/lance/memwal/ShardWriter.java
index a5a3000bdba..da4c621e934 100644
--- a/java/src/main/java/org/lance/memwal/ShardWriter.java
+++ b/java/src/main/java/org/lance/memwal/ShardWriter.java
@@ -36,6 +36,7 @@
  * <pre>{@code
  * try (ShardWriter writer = dataset.memWalWriter(shardId)) {
  *   writer.put(reader);
+ *   writer.delete(keys);
  * }
  * }</pre>
  *
@@ -99,6 +100,34 @@ public void put(ArrowReader reader) {
 
   private native void nativePut(long streamAddress);
 
+  /**
+   * Delete rows from the MemWAL by primary key.
+   *
+   * <p>Each batch in {@code reader} must carry this shard's primary key column(s); other columns
+   * are ignored. Lance builds a tombstone row per key — the primary key plus {@code _tombstone =
+   * true} and null in every other column — and appends it like an ordinary write. The tombstone is
+   * the newest value for its key: it wins newest-per-PK resolution (suppressing the older real row)
+   * and is then dropped from query results.
+   *
+   * <p>Only supported in memtable mode. Because a tombstone nulls every non-PK column, those
+   * columns must be nullable in the base schema; deleting against a schema with a non-nullable
+   * non-PK column errors. Deleting on a shard with no primary key columns also errors.
+   *
+   * @param reader the keys to delete; consumed fully by this call
+   */
+  public void delete(ArrowReader reader) {
+    Preconditions.checkNotNull(reader, "reader must not be null");
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeShardWriterHandle != 0, "ShardWriter is closed");
+      try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) {
+        Data.exportArrayStream(allocator, reader, stream);
+        nativeDelete(stream.memoryAddress());
+      }
+    }
+  }
+
+  private native void nativeDelete(long streamAddress);
+
   /** Return a snapshot of cumulative write statistics. */
   public WriteStats stats() {
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
@@ -122,9 +151,8 @@ public MemTableStats memtableStats() {
   /**
    * Create an LSM scanner that includes this writer's active MemTable.
    *
-   * <p>The scanner covers the base table, the given flushed generations, and the current active
-   * MemTable, providing read-your-writes consistency. This writer's own shard is included
-   * automatically.
+   * <p>The scanner covers the base table, the given SSTables, and the current active MemTable,
+   * providing read-your-writes consistency. This writer's own shard is included automatically.
    *
    * @param shardSnapshots snapshots of other shards to include
    * @return an LSM scanner
diff --git a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
index 40310907d3e..d6bf2aeb383 100644
--- a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
+++ b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
@@ -28,7 +28,6 @@
  */
 public class ShardWriterConfig {
   private Optional<Boolean> durableWrite = Optional.empty();
-  private Optional<Boolean> syncIndexedWrite = Optional.empty();
   private Optional<Long> maxWalBufferSize = Optional.empty();
   private Optional<Long> maxWalFlushIntervalMs = Optional.empty();
   private Optional<Long> maxMemtableSize = Optional.empty();
@@ -36,8 +35,6 @@ public class ShardWriterConfig {
   private Optional<Long> maxMemtableBatches = Optional.empty();
   private Optional<Long> maxUnflushedMemtableBytes = Optional.empty();
   private Optional<Long> manifestScanBatchSize = Optional.empty();
-  private Optional<Long> asyncIndexBufferRows = Optional.empty();
-  private Optional<Long> asyncIndexIntervalMs = Optional.empty();
   private Optional<Long> backpressureLogIntervalMs = Optional.empty();
   private Optional<Long> statsLogIntervalMs = Optional.empty();
   private List<MemWalHnswParams> hnswParams = Collections.emptyList();
@@ -48,12 +45,6 @@ public ShardWriterConfig withDurableWrite(boolean durableWrite) {
     return this;
   }
 
-  /** Whether indexed writes are applied synchronously. */
-  public ShardWriterConfig withSyncIndexedWrite(boolean syncIndexedWrite) {
-    this.syncIndexedWrite = Optional.of(syncIndexedWrite);
-    return this;
-  }
-
   /** Maximum size of the in-memory WAL buffer, in bytes. */
   public ShardWriterConfig withMaxWalBufferSize(long maxWalBufferSize) {
     Preconditions.checkArgument(
@@ -118,26 +109,6 @@ public ShardWriterConfig withManifestScanBatchSize(long manifestScanBatchSize) {
     return this;
   }
 
-  /** Number of rows buffered before an asynchronous index update is triggered. */
-  public ShardWriterConfig withAsyncIndexBufferRows(long asyncIndexBufferRows) {
-    Preconditions.checkArgument(
-        asyncIndexBufferRows >= 0,
-        "asyncIndexBufferRows must not be negative, got %s",
-        asyncIndexBufferRows);
-    this.asyncIndexBufferRows = Optional.of(asyncIndexBufferRows);
-    return this;
-  }
-
-  /** Interval between asynchronous index updates, in milliseconds. */
-  public ShardWriterConfig withAsyncIndexIntervalMs(long asyncIndexIntervalMs) {
-    Preconditions.checkArgument(
-        asyncIndexIntervalMs >= 0,
-        "asyncIndexIntervalMs must not be negative, got %s",
-        asyncIndexIntervalMs);
-    this.asyncIndexIntervalMs = Optional.of(asyncIndexIntervalMs);
-    return this;
-  }
-
   /** Interval between backpressure log messages, in milliseconds. */
   public ShardWriterConfig withBackpressureLogIntervalMs(long backpressureLogIntervalMs) {
     Preconditions.checkArgument(
@@ -172,10 +143,6 @@ public Optional<Boolean> durableWrite() {
     return durableWrite;
   }
 
-  public Optional<Boolean> syncIndexedWrite() {
-    return syncIndexedWrite;
-  }
-
   public Optional<Long> maxWalBufferSize() {
     return maxWalBufferSize;
   }
@@ -204,14 +171,6 @@ public Optional<Long> manifestScanBatchSize() {
     return manifestScanBatchSize;
   }
 
-  public Optional<Long> asyncIndexBufferRows() {
-    return asyncIndexBufferRows;
-  }
-
-  public Optional<Long> asyncIndexIntervalMs() {
-    return asyncIndexIntervalMs;
-  }
-
   public Optional<Long> backpressureLogIntervalMs() {
     return backpressureLogIntervalMs;
   }
diff --git a/java/src/main/java/org/lance/memwal/FlushedGeneration.java b/java/src/main/java/org/lance/memwal/SsTable.java
similarity index 78%
rename from java/src/main/java/org/lance/memwal/FlushedGeneration.java
rename to java/src/main/java/org/lance/memwal/SsTable.java
index 66288161438..55cb2e63f77 100644
--- a/java/src/main/java/org/lance/memwal/FlushedGeneration.java
+++ b/java/src/main/java/org/lance/memwal/SsTable.java
@@ -15,22 +15,22 @@
 
 import com.google.common.base.MoreObjects;
 
-/** A flushed MemWAL generation and the storage path of its Lance files. */
-public class FlushedGeneration {
+/** An SSTable and the storage path of its Lance files. */
+public class SsTable {
   private final long generation;
   private final String path;
 
-  public FlushedGeneration(long generation, String path) {
+  public SsTable(long generation, String path) {
     this.generation = generation;
     this.path = path;
   }
 
-  /** The generation number of this flushed MemTable. */
+  /** The generation number of this SSTable. */
   public long generation() {
     return generation;
   }
 
-  /** The storage path of the flushed Lance files. */
+  /** The storage path of the SSTable Lance files. */
   public String path() {
     return path;
   }
diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java
index 2ae27b67cba..a6157bb5ddb 100644
--- a/java/src/main/java/org/lance/merge/MergeInsertParams.java
+++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java
@@ -13,7 +13,7 @@
  */
 package org.lance.merge;
 
-import org.lance.memwal.MergedGeneration;
+import org.lance.memwal.CompactedSsTable;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
@@ -39,7 +39,7 @@ public class MergeInsertParams {
   private long retryTimeoutMs = 30 * 1000;
   private boolean skipAutoCleanup = false;
   private boolean useIndex = true;
-  private List<MergedGeneration> markedGenerations = Collections.emptyList();
+  private List<CompactedSsTable> compactedSstables = Collections.emptyList();
 
   public MergeInsertParams(List<String> on) {
     this.on = on;
@@ -245,17 +245,17 @@ public MergeInsertParams withUseIndex(boolean useIndex) {
   }
 
   /**
-   * Mark MemWAL generations as merged into the base table.
+   * Mark MemWAL SSTables as compacted into the base table.
    *
-   * <p>Use this when the merge insert incorporates data from MemWAL flushed generations. It updates
-   * the MemWAL generation tracking to prevent the same generations from being merged again.
+   * <p>Use this when merge insert compacts MemWAL SSTables. It updates MemWAL compaction progress
+   * to prevent the same SSTables from being compacted again.
    *
-   * @param generations the flushed generations being merged
+   * @param sstables the SSTables being compacted
    * @return This MergeInsertParams instance
    */
-  public MergeInsertParams markGenerationsAsMerged(List<MergedGeneration> generations) {
-    Preconditions.checkNotNull(generations, "generations must not be null");
-    this.markedGenerations = generations;
+  public MergeInsertParams markSstablesAsCompacted(List<CompactedSsTable> sstables) {
+    Preconditions.checkNotNull(sstables, "sstables must not be null");
+    this.compactedSstables = sstables;
     return this;
   }
 
@@ -263,8 +263,8 @@ public List<String> on() {
     return on;
   }
 
-  public List<MergedGeneration> markedGenerations() {
-    return markedGenerations;
+  public List<CompactedSsTable> getCompactedSstables() {
+    return compactedSstables;
   }
 
   public WhenMatched whenMatched() {
diff --git a/java/src/test/java/org/lance/DatasetTest.java b/java/src/test/java/org/lance/DatasetTest.java
index 45466a0367c..5999cbc0b19 100644
--- a/java/src/test/java/org/lance/DatasetTest.java
+++ b/java/src/test/java/org/lance/DatasetTest.java
@@ -80,6 +80,7 @@
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -2026,6 +2027,16 @@ void testReadZeroLengthBlob(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  void testTakeNullBlobPreservesSelection(@TempDir Path tempDir) throws Exception {
+    String base = tempDir.resolve("testTakeNullBlobPreservesSelection").toString();
+    try (Dataset ds = TestUtils.createBlobDataset(base, 128, 8)) {
+      List<BlobFile> blobs = ds.takeBlobsByIndices(Collections.singletonList(15L), "blobs");
+      assertEquals(1, blobs.size());
+      assertNull(blobs.get(0));
+    }
+  }
+
   @Test
   void testReadLargeBlobAndRanges(@TempDir Path tempDir) throws Exception {
     String base = tempDir.resolve("testReadLargeBlobAndRanges").toString();
diff --git a/java/src/test/java/org/lance/FileReaderWriterTest.java b/java/src/test/java/org/lance/FileReaderWriterTest.java
index a849a87c576..85c7430f087 100644
--- a/java/src/test/java/org/lance/FileReaderWriterTest.java
+++ b/java/src/test/java/org/lance/FileReaderWriterTest.java
@@ -256,7 +256,9 @@ void testInvalidPath() {
       LanceFileReader.open("/tmp/does_not_exist.lance", allocator);
       fail("Expected LanceException to be thrown");
     } catch (IOException e) {
-      assertTrue(e.getMessage().contains("Object at location /tmp/does_not_exist.lance not found"));
+      String message = e.getMessage();
+      assertTrue(message.contains("/tmp/does_not_exist.lance"));
+      assertTrue(message.toLowerCase().contains("not found"));
     }
     try {
       LanceFileReader.open("", allocator);
diff --git a/java/src/test/java/org/lance/ScannerTest.java b/java/src/test/java/org/lance/ScannerTest.java
index 00434034b64..0b07026bc68 100644
--- a/java/src/test/java/org/lance/ScannerTest.java
+++ b/java/src/test/java/org/lance/ScannerTest.java
@@ -422,25 +422,32 @@ void testDatasetScannerBatchReadahead(@TempDir Path tempDir) throws Exception {
       TestUtils.SimpleTestDataset testDataset =
           new TestUtils.SimpleTestDataset(allocator, datasetPath);
       testDataset.createEmptyDataset().close();
-      int totalRows = 1000;
-      int batchSize = 100;
-      int batchReadahead = 5;
-      try (Dataset dataset = testDataset.write(1, totalRows)) {
+
+      int totalRows = 2000;
+      int maxRowsPerFile = 100; // ~20 fragments
+      List<FragmentMetadata> fragments = testDataset.createNewFragment(totalRows, maxRowsPerFile);
+      assertTrue(fragments.size() > 1, "expected multiple fragments, got " + fragments.size());
+
+      FragmentOperation.Append append = new FragmentOperation.Append(fragments);
+      try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) {
+        int batchReadahead = 2; // far below the default (num compute CPUs)
         try (LanceScanner scanner =
             dataset.newScan(
-                new ScanOptions.Builder()
-                    .batchSize(batchSize)
-                    .batchReadahead(batchReadahead)
-                    .build())) {
-          // This test is more about ensuring that the batchReadahead parameter is accepted
-          // and doesn't cause errors. The actual effect of batchReadahead might not be
-          // directly observable in this test.
+                new ScanOptions.Builder().batchSize(50).batchReadahead(batchReadahead).build())) {
           try (ArrowReader reader = scanner.scanBatches()) {
             int rowCount = 0;
+            long idSum = 0;
             while (reader.loadNextBatch()) {
-              rowCount += reader.getVectorSchemaRoot().getRowCount();
+              VectorSchemaRoot root = reader.getVectorSchemaRoot();
+              IntVector ids = (IntVector) root.getVector("id");
+              for (int i = 0; i < root.getRowCount(); i++) {
+                idSum += ids.get(i);
+              }
+              rowCount += root.getRowCount();
             }
             assertEquals(totalRows, rowCount);
+            // ids are the contiguous range [0, totalRows)
+            assertEquals((long) totalRows * (totalRows - 1) / 2, idSum);
           }
         }
       }
diff --git a/java/src/test/java/org/lance/SessionTest.java b/java/src/test/java/org/lance/SessionTest.java
index b2ed3baa343..9f953ab821e 100644
--- a/java/src/test/java/org/lance/SessionTest.java
+++ b/java/src/test/java/org/lance/SessionTest.java
@@ -259,6 +259,49 @@ void testSessionToString() {
     assertEquals("Session(closed)", closedSession.toString());
   }
 
+  @Test
+  void testMetadataCacheStats(@TempDir Path tempDir) {
+    String datasetPath = tempDir.resolve("dataset_metadata_cache_stats").toString();
+
+    try (BufferAllocator allocator = new RootAllocator();
+        Session session = Session.builder().build()) {
+      CacheStats initialStats = session.metadataCacheStats();
+      assertEquals(0, initialStats.getHits());
+      assertEquals(0, initialStats.getMisses());
+      assertEquals(0, initialStats.getNumEntries());
+      assertEquals(0, initialStats.getSizeBytes());
+      assertEquals(0.0, initialStats.getHitRatio());
+
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+
+      // First open populates the metadata cache, producing misses
+      try (Dataset ds =
+          Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) {
+        ds.countRows();
+      }
+      CacheStats statsAfterFirstOpen = session.metadataCacheStats();
+      assertTrue(statsAfterFirstOpen.getMisses() > 0);
+      assertTrue(statsAfterFirstOpen.getNumEntries() > 0);
+      assertTrue(statsAfterFirstOpen.getSizeBytes() > 0);
+
+      // Reopening the same dataset should hit the shared metadata cache
+      try (Dataset ds =
+          Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) {
+        ds.countRows();
+      }
+      CacheStats statsAfterSecondOpen = session.metadataCacheStats();
+      assertTrue(statsAfterSecondOpen.getHits() > statsAfterFirstOpen.getHits());
+      assertTrue(statsAfterSecondOpen.getHitRatio() > 0.0);
+
+      // Stats are not accessible once the session is closed (close is idempotent,
+      // so the implicit close from try-with-resources remains safe)
+      session.close();
+      assertThrows(IllegalArgumentException.class, session::metadataCacheStats);
+    }
+  }
+
   @Test
   void testInvalidCacheSizes() {
     assertThrows(
diff --git a/java/src/test/java/org/lance/TestUtils.java b/java/src/test/java/org/lance/TestUtils.java
index c9033361103..46c60c777cb 100644
--- a/java/src/test/java/org/lance/TestUtils.java
+++ b/java/src/test/java/org/lance/TestUtils.java
@@ -694,7 +694,8 @@ public Dataset createEmptyDataset() {
     /**
      * Create a single fragment with given row count and return its metadata. The fragment contains
      * deterministic blob payloads: - Every 16th row starting at 0 has zero-length blob - Every 16th
-     * row starting at 1 has a ~1 MiB payload - Others have small variable blobs (128..383 bytes)
+     * row starting at 1 has a ~1 MiB payload - Every 16th row starting at 15 is null - Others have
+     * small variable blobs (128..383 bytes)
      */
     public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) {
       Preconditions.checkArgument(rowCount >= 0, "rowCount must be non-negative");
@@ -716,6 +717,8 @@ public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) {
             byte[] big = new byte[1024 * 1024];
             Arrays.fill(big, (byte) 0xAB);
             blobsVec.setSafe(i, big);
+          } else if (i % 16 == 15) {
+            blobsVec.setNull(i);
           } else {
             // small variable blob
             int sz = 128 + (i % 256);
diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
index e5024a95c2a..2cbacb55bfe 100644
--- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
+++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
@@ -13,19 +13,82 @@
  */
 package org.lance.index.scalar;
 
+import org.lance.util.JsonUtils;
+
 import org.junit.jupiter.api.Test;
 
+import java.util.Map;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-public class InvertedIndexParamsTest {
+class InvertedIndexParamsTest {
 
   @Test
-  public void testIcuSplitTokenizerVariant() {
+  void testIcuSplitTokenizerVariant() {
     ScalarIndexParams params = InvertedIndexParams.builder().baseTokenizer("icu/split").build();
 
     assertEquals("inverted", params.getIndexType());
     String jsonParams = params.getJsonParams().orElseThrow(AssertionError::new);
     assertTrue(jsonParams.contains("\"base_tokenizer\":\"icu/split\""));
   }
+
+  @Test
+  void defaultBlockSizeIsSerialized() {
+    ScalarIndexParams params = InvertedIndexParams.builder().build();
+
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(128, ((Number) json.get("block_size")).intValue());
+  }
+
+  @Test
+  void blockSizeIsSerialized() {
+    ScalarIndexParams params = InvertedIndexParams.builder().blockSize(128).build();
+
+    assertEquals("inverted", params.getIndexType());
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(128, ((Number) json.get("block_size")).intValue());
+  }
+
+  @Test
+  void invalidBlockSizeIsRejected() {
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(129));
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(512));
+  }
+
+  @Test
+  void formatVersionThreeSupportsBothBlockSizes() {
+    for (int blockSize : new int[] {128, 256}) {
+      ScalarIndexParams params =
+          InvertedIndexParams.builder().blockSize(blockSize).formatVersion(3).build();
+      Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+      assertEquals(blockSize, ((Number) json.get("block_size")).intValue());
+      assertEquals(3, ((Number) json.get("format_version")).intValue());
+    }
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build());
+  }
+
+  @Test
+  void formatVersionFourIsRejected() {
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().formatVersion(4));
+  }
+
+  @Test
+  void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() {
+    ScalarIndexParams params =
+        InvertedIndexParams.builder().baseTokenizer("code").formatVersion(3).build();
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(3, ((Number) json.get("format_version")).intValue());
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build());
+  }
 }
diff --git a/java/src/test/java/org/lance/memwal/MemWalTest.java b/java/src/test/java/org/lance/memwal/MemWalTest.java
index de0124bb1b3..e9bd053f298 100644
--- a/java/src/test/java/org/lance/memwal/MemWalTest.java
+++ b/java/src/test/java/org/lance/memwal/MemWalTest.java
@@ -97,6 +97,22 @@ private static VectorSchemaRoot lookupRoot(BufferAllocator allocator, long[] ids
     return root;
   }
 
+  /** Build a single-batch root carrying only the {@code id} primary key, for deletes. */
+  private static VectorSchemaRoot keysRoot(BufferAllocator allocator, long[] ids) {
+    VectorSchemaRoot root =
+        VectorSchemaRoot.create(
+            new Schema(
+                Collections.singletonList(Field.nullable("id", new ArrowType.Int(64, true)))),
+            allocator);
+    BigIntVector idVector = (BigIntVector) root.getVector("id");
+    idVector.allocateNew(ids.length);
+    for (int i = 0; i < ids.length; i++) {
+      idVector.set(i, ids[i]);
+    }
+    root.setRowCount(ids.length);
+    return root;
+  }
+
   /** Build a single-batch append-only root without primary-key metadata. */
   private static VectorSchemaRoot appendOnlyRoot(
       BufferAllocator allocator, long[] ids, String prefix) {
@@ -144,12 +160,12 @@ private static Dataset writeAppendOnlyDataset(
   }
 
   /**
-   * Stage a <em>faithful</em> flushed generation at {@code genPath}: the Lance dataset plus its
-   * primary-key dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM
-   * scanner's cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not
-   * a state production produces. Mirrors the Python {@code _write_flushed_gen} test helper.
+   * Stage a <em>faithful</em> SSTable at {@code genPath}: the Lance dataset plus its primary-key
+   * dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM scanner's
+   * cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not a state
+   * production produces. Mirrors the Python {@code _write_sstable} test helper.
    */
-  private static void writeFlushedGen(
+  private static void writeSsTable(
       BufferAllocator allocator, String genPath, long[] ids, String prefix) throws Exception {
     writeLookupDataset(allocator, genPath, ids, prefix).close();
     try (VectorSchemaRoot root = lookupRoot(allocator, ids, prefix);
@@ -161,8 +177,8 @@ private static void writeFlushedGen(
   }
 
   /**
-   * Test-support native: write the primary-key dedup sidecar for a flushed-generation dataset
-   * already staged at {@code genPath}. See {@link #writeFlushedGen}.
+   * Test-support native: write the primary-key dedup sidecar for an SSTable dataset already staged
+   * at {@code genPath}. See {@link #writeSsTable}.
    */
   private static native void nativeWritePkSidecar(
       String genPath, long streamAddress, List<String> pkColumns);
@@ -382,6 +398,51 @@ void testShardWriterPutAndLsmScanner(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  void testShardWriterDeleteMasksBaseRow(@TempDir Path tempDir) throws Exception {
+    String path = tempDir.resolve("base").toString();
+    String shardId = UUID.randomUUID().toString();
+    try (BufferAllocator allocator = new RootAllocator();
+        Dataset dataset = writeLookupDataset(allocator, path, new long[] {1, 2, 3}, "base")) {
+      dataset.initializeMemWal(new InitializeMemWalParams());
+
+      ShardWriterConfig config =
+          new ShardWriterConfig()
+              .withDurableWrite(true)
+              .withMaxWalBufferSize(1)
+              .withMaxWalFlushIntervalMs(10);
+
+      try (ShardWriter writer = dataset.memWalWriter(shardId, config)) {
+        try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {4}, "writer");
+            ArrowReader reader = toReader(allocator, root)) {
+          writer.put(reader);
+        }
+        try (VectorSchemaRoot keys = keysRoot(allocator, new long[] {2});
+            ArrowReader reader = toReader(allocator, keys)) {
+          writer.delete(reader);
+        }
+
+        Map<Long, String> byId = Collections.emptyMap();
+        long deadline = System.currentTimeMillis() + 10_000;
+        while (System.currentTimeMillis() < deadline) {
+          try (LsmScanner scanner = writer.lsmScanner();
+              ArrowReader reader = scanner.scanBatches()) {
+            byId = readByName(reader);
+          }
+          if (!byId.containsKey(2L) && "writer_4".equals(byId.get(4L))) {
+            break;
+          }
+          Thread.sleep(50);
+        }
+
+        assertEquals("base_1", byId.get(1L));
+        assertFalse(byId.containsKey(2L), "deleted base row should be masked by the tombstone");
+        assertEquals("base_3", byId.get(3L));
+        assertEquals("writer_4", byId.get(4L));
+      }
+    }
+  }
+
   @Test
   void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
     String basePath = tempDir.resolve("base").toString();
@@ -390,12 +451,12 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
         Dataset dataset = writeLookupDataset(allocator, basePath, new long[] {1, 2, 3}, "base")) {
       dataset.initializeMemWal(new InitializeMemWalParams());
 
-      // Flushed generation overwrites id=2.
+      // SSTable overwrites id=2.
       String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1";
-      writeFlushedGen(allocator, genPath, new long[] {2}, "gen1");
+      writeSsTable(allocator, genPath, new long[] {2}, "gen1");
 
       ShardSnapshot snapshot =
-          new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2);
+          new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2);
 
       try (LsmScanner scanner =
               LsmScanner.fromSnapshots(dataset, Collections.singletonList(snapshot));
@@ -403,7 +464,7 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
         Map<Long, String> byId = readByName(reader);
         assertEquals(3, byId.size(), "Expected 3 deduplicated rows");
         assertEquals("base_1", byId.get(1L));
-        assertEquals("gen1_2", byId.get(2L), "Flushed generation must win over base");
+        assertEquals("gen1_2", byId.get(2L), "SSTable must win over base");
         assertEquals("base_3", byId.get(3L));
       }
 
@@ -426,14 +487,14 @@ void testPointLookup(@TempDir Path tempDir) throws Exception {
       dataset.initializeMemWal(new InitializeMemWalParams());
 
       String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1";
-      writeFlushedGen(allocator, genPath, new long[] {2}, "gen1");
+      writeSsTable(allocator, genPath, new long[] {2}, "gen1");
 
       ShardSnapshot snapshot =
-          new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2);
+          new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2);
 
       try (LsmPointLookupPlanner planner =
           new LsmPointLookupPlanner(dataset, Collections.singletonList(snapshot))) {
-        // id=2 must resolve to the flushed-generation value.
+        // id=2 must resolve to the SSTable value.
         assertEquals("gen1_2", lookup(planner, allocator, 2L));
         // id=1 only exists in the base table.
         assertEquals("base_1", lookup(planner, allocator, 1L));
@@ -459,7 +520,7 @@ private static String lookup(LsmPointLookupPlanner planner, BufferAllocator allo
   }
 
   @Test
-  void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Exception {
+  void testMergeInsertMarkSstablesAsCompacted(@TempDir Path tempDir) throws Exception {
     String path = tempDir.resolve("base").toString();
     String shardId = UUID.randomUUID().toString();
     try (BufferAllocator allocator = new RootAllocator()) {
@@ -471,8 +532,8 @@ void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Except
             new MergeInsertParams(Collections.singletonList("id"))
                 .withMatchedUpdateAll()
                 .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll)
-                .markGenerationsAsMerged(
-                    Collections.singletonList(new MergedGeneration(shardId, 1)));
+                .markSstablesAsCompacted(
+                    Collections.singletonList(new CompactedSsTable(shardId, 1)));
 
         try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {2, 4}, "merged");
             ArrowReader reader = toReader(allocator, root);
diff --git a/protos/AGENTS.md b/protos/AGENTS.md
index 23aef9fc196..2dba0e23dcb 100644
--- a/protos/AGENTS.md
+++ b/protos/AGENTS.md
@@ -4,7 +4,8 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 
 ## Compatibility
 
-- All changes must be backwards compatible. Never re-use or change field numbers of existing fields.
+- Protobuf schemas that are part of a stable file format or any other stable persisted contract must remain backwards compatible. Never reuse or change their existing field numbers.
+- Protobuf schemas used exclusively by an unstable file format follow the root file-format stability contract: do not preserve compatibility with prior unstable revisions. Before making a breaking protobuf change, verify that the schema is not shared with a stable format or another persisted contract.
 
 ## Schema Design
 
diff --git a/protos/encodings_v2_1.proto b/protos/encodings_v2_1.proto
index 46fd012fb58..51427332063 100644
--- a/protos/encodings_v2_1.proto
+++ b/protos/encodings_v2_1.proto
@@ -147,6 +147,128 @@ message FullZipLayout {
   repeated RepDefLayer layers = 8;
 }
 
+// A layout used for sparse flat or nested pages where Arrow structure is represented directly
+// in layer-local slot domains instead of as dense repetition / definition events.
+//
+// Structural layers are ordered from outer-most to inner-most. Values remain mini-block
+// compressed and are split into independently readable chunks.
+message SparseLayout {
+  // Description of the compression of values.
+  CompressiveEncoding value_compression = 1;
+  // Number of value buffers in each mini-block chunk. This does not include structural buffers.
+  uint64 num_buffers = 2;
+  // Number of entries in the equivalent dense repetition / definition stream. This equals
+  // num_visible_items plus one structural placeholder for every list slot without children.
+  // Null leaf slots count as visible items because they still occupy positions in Arrow's
+  // leaf value buffer. For example, a nullable primitive with 100 slots, 30 of them null,
+  // has num_items = num_visible_items = 100.
+  uint64 num_items = 3;
+  // Number of leaf value slots encoded in the value chunks, including null leaf slots.
+  uint64 num_visible_items = 4;
+  // If true, chunk-local value buffer sizes use u32. Otherwise they use u16.
+  bool has_large_chunk = 5;
+  // Structural layers ordered from outer-most to inner-most. This may be empty for a flat,
+  // non-nullable leaf page whose scheduling domain equals num_visible_items.
+  repeated SparseStructuralLayer structural_layers = 6;
+}
+
+// A domain is a layer-local integer coordinate space [0, num_slots). A slot is one
+// element in that space. The outer-most domain is the page's top-level rows; each
+// layer's child domain is the next layer's parent domain, and the terminal child
+// domain contains num_visible_items leaf value slots.
+message SparseStructuralLayer {
+  // Exactly one layer kind is required.
+  oneof layer {
+    SparseValidityLayer validity = 1;
+    SparseListLayer list = 2;
+    SparseFixedSizeListLayer fixed_size_list = 3;
+  }
+}
+
+message SparseValidityLayer {
+  // Number of nullable item or struct slots in this layer's parent and child domain.
+  uint64 num_slots = 1;
+  // Validity for the slots in this layer.
+  SparseValiditySet validity = 2;
+}
+
+message SparseListLayer {
+  // Number of list, large-list, or map slots in this layer's parent domain.
+  uint64 num_slots = 1;
+  // Number of slots in this layer's child domain.
+  uint64 num_child_slots = 2;
+  // Non-empty parent slots. Valid parent slots absent from this set are empty lists.
+  SparsePositionSet non_empty_positions = 3;
+  // Positive child counts corresponding one-for-one with non_empty_positions.
+  SparseCountSet counts = 4;
+  // Validity for the parent slots in this layer.
+  SparseValiditySet validity = 5;
+}
+
+message SparseFixedSizeListLayer {
+  // Number of fixed-size-list slots in this layer's parent domain.
+  uint64 num_slots = 1;
+  // Number of children per parent slot. The child domain has num_slots * dimension slots.
+  uint64 dimension = 2;
+  // Validity for the parent slots in this layer.
+  SparseValiditySet validity = 3;
+}
+
+message SparseValiditySet {
+  enum Meaning {
+    SPARSE_VALIDITY_UNSPECIFIED = 0;
+    // Stored positions are null; all other positions are valid.
+    SPARSE_VALIDITY_NULL_POSITIONS = 1;
+    // Stored positions are valid; all other positions are null.
+    SPARSE_VALIDITY_VALID_POSITIONS = 2;
+  }
+
+  Meaning meaning = 1;
+  SparsePositionSet positions = 2;
+}
+
+message SparsePositionEmpty {}
+
+message SparsePositionAll {}
+
+message SparsePositionRange {
+  uint64 start = 1;
+  uint64 length = 2;
+}
+
+message SparsePositionSet {
+  oneof positions {
+    // Delta-compressed u64 positions. Cardinality is num_positions.
+    CompressiveEncoding explicit = 1;
+    // One contiguous, non-empty range.
+    SparsePositionRange range = 2;
+    // Every position in the domain.
+    SparsePositionAll all = 3;
+    // No positions in the domain.
+    SparsePositionEmpty empty = 4;
+  }
+  // Semantic cardinality of this set.
+  uint64 num_positions = 5;
+}
+
+message SparseCountEmpty {}
+
+message SparseCountConstant {
+  // Child count shared by every non-empty list slot.
+  uint64 value = 1;
+}
+
+message SparseCountSet {
+  oneof counts {
+    // Compressed u64 child counts. Cardinality comes from the containing position set.
+    CompressiveEncoding explicit = 1;
+    // One positive child count shared by every non-empty list slot.
+    SparseCountConstant constant = 2;
+    // No counts; valid only when there are no non-empty list slots.
+    SparseCountEmpty empty = 3;
+  }
+}
+
 // A layout used for pages where all (visible) values are the same scalar value.
 //
 // This generalizes the prior AllNullLayout semantics for file_version >= 2.2.
@@ -206,6 +328,8 @@ message PageLayout {
     // A layout where large binary data is encoded externally
     // and only the descriptions are put in the page
     BlobLayout blob_layout = 4;
+    // A sparse structural layout. This variant requires file version 2.3 or later.
+    SparseLayout sparse_layout = 5;
   }
 }
 
diff --git a/protos/file2.proto b/protos/file2.proto
index da0b1d5e96c..650a1568da6 100644
--- a/protos/file2.proto
+++ b/protos/file2.proto
@@ -207,4 +207,4 @@ message ColumnMetadata {
 //
 // This file format is extremely minimal.  It is a building block for
 // creating more useful readers and writers and not terribly useful by itself.
-// Other protobuf files will describe how this can be extended.
\ No newline at end of file
+// Other protobuf files will describe how this can be extended.
diff --git a/protos/index_old.proto b/protos/index_old.proto
index 601aa2681da..bdba4ad0252 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -24,8 +24,44 @@ message BTreeIndexDetails {}
 message BitmapIndexDetails {}
 message LabelListIndexDetails {}
 message NGramIndexDetails {}
-message ZoneMapIndexDetails {}
+message ZoneMapIndexDetails {
+  // Number of rows per zone. Optional for backwards compatibility: absent on
+  // datasets written before this field was added. When absent, no seed writer
+  // is created for the index.
+  optional uint64 rows_per_zone = 1;
+  // Whether seed-based incremental updates are enabled for this index.
+  // On-disk semantics: absent means seeds are disabled (old datasets written
+  // before this field was added). Present false means explicitly disabled.
+  // Present true means seeds are enabled: the index will embed per-fragment
+  // seed buffers in data files and harvest them during incremental updates
+  // to skip full column scans.
+  // Creation-time default: index creation code sets this to true for
+  // variable-length types (strings, binary) and fixed-width types wider than
+  // 8 bytes, and to false for narrow fixed-width types (e.g. Int64, Float64).
+  optional bool use_seeds = 2;
+}
 message InvertedIndexDetails {
+  message CodeTokenizerConfig {
+    // Split one lexical identifier into subwords, e.g. getUserName ->
+    // get/user/name.
+    bool split_identifiers = 1;
+    // Split identifier subwords across letter/number boundaries, e.g.
+    // HTML2JSON -> html/2/json. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool split_on_numerics = 2;
+    // Keep the complete lexical identifier in addition to subwords, e.g.
+    // user_name plus user/name. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool preserve_original = 3;
+    // Index operator tokens such as "::", "->", and "!=". Operators are not
+    // indexed by default because they are often high-frequency noise.
+    bool index_operators = 4;
+  }
+
+  // Lexical tokenizer used after document-level text extraction. This is an
+  // implementation component such as "simple", "icu", "ngram", or "code".
+  // Input-time analyzer profiles are expanded into this field and the concrete
+  // options below before these details are persisted.
   // Marking this field as optional as old versions of the index store blank details and we
   // need to make sure we have a proper optional field to detect this.
   optional string base_tokenizer = 1;
@@ -39,4 +75,13 @@ message InvertedIndexDetails {
   uint32 min_ngram_length = 9;
   uint32 max_ngram_length = 10;
   bool prefix_only = 11;
+  // Number of documents per compressed posting block. An absent value means
+  // the index predates this field and must use the legacy block size of 128.
+  // A present value records the block size used by the index; 256 is valid
+  // with format versions 3 and 4.
+  optional uint32 block_size = 12;
+  // Options for base_tokenizer = "code". Presence records the code tokenizer
+  // configuration used to build the index; absence means there is no
+  // code-specific configuration to apply.
+  CodeTokenizerConfig code_config = 13;
 }
diff --git a/protos/table.proto b/protos/table.proto
index 8d0cb249fda..9a64230f40f 100644
--- a/protos/table.proto
+++ b/protos/table.proto
@@ -115,6 +115,9 @@ message Manifest {
   // * 1 << 3: table config is present
   // * 1 << 4: dataset uses multiple base paths
   // * 1 << 5: transaction file writes are disabled
+  // * 1 << 6: data overlay files are present (see DataOverlayFile). Readers that do
+  //   not understand overlays must refuse the dataset, since ignoring an overlay
+  //   would silently return stale base values.
   uint64 reader_feature_flags = 9;
 
   // Feature flags for writers.
@@ -313,6 +316,15 @@ message DataFragment {
 
   repeated DataFile files = 2;
 
+  // Optional overlay files for this fragment, which supply new values for a
+  // subset of cells without rewriting the base data files. This MUST be empty
+  // if the data overlay files feature flag (64) is not set in the manifest.
+  //
+  // Order is significant: a later entry is newer than an earlier one. When two
+  // overlays cover the same (offset, field) and share a `committed_version`, the
+  // later entry wins. See DataOverlayFile for the full resolution rules.
+  repeated DataOverlayFile overlays = 11;
+
   // File that indicates which rows, if any, should be considered deleted.
   DeletionFile deletion_file = 3;
 
@@ -435,6 +447,66 @@ message DataFile {
   optional uint32 base_id = 7;
 } // DataFile
 
+// An overlay file supplies new values for a subset of (row offset, field) cells
+// within a fragment, without rewriting the fragment's base data files. It is
+// used for efficient updates when only a small fraction of rows and/or columns
+// change.
+//
+// On read, a cell is resolved by consulting the fragment's overlays from newest
+// to oldest: the first overlay that covers that (offset, field) wins; if none
+// cover it, the value falls through to the base data file. Because deletions
+// take precedence over overlays, an overlay value for an offset that is also
+// marked deleted is dead and is ignored.
+//
+// The overlay's data file does NOT store a row-offset key column. Within a value
+// column, the position of a covered offset's value is the rank (0-based count of
+// set bits below it) of that offset within the field's coverage bitmap. Because
+// fields may cover different offset sets, the value columns of a single overlay
+// data file may have different lengths (which the Lance file format permits).
+message DataOverlayFile {
+  // The data file storing the overlay's new cell values, one value column per
+  // field in `data_file.fields`. No row-offset key column is stored.
+  DataFile data_file = 1;
+
+  // Which (offset, field) cells this overlay provides values for.
+  oneof coverage {
+    // A single 32-bit Roaring bitmap of physical row offsets that applies to
+    // every field in `data_file.fields` (a "dense" / rectangular overlay).
+    // Every covered offset has a value for every field. This is the common case
+    // for a plain UPDATE, where one SET list is applied to one set of rows.
+    bytes shared_offset_bitmap = 2;
+    // Per-field coverage for a "sparse" overlay, used when different fields cover
+    // different offset sets (e.g. a MERGE with multiple WHEN MATCHED branches).
+    FieldCoverage field_coverage = 4;
+  }
+
+  // The dataset version at which this overlay became effective: the version of
+  // the commit that introduced it, NOT the version it was read from. It is
+  // stamped at commit time and re-stamped if the commit is retried, in the same
+  // way as the created-at / last-updated-at version sequences.
+  //
+  // This drives two orderings:
+  //  * Versus index builds: an index whose `dataset_version` >= this value
+  //    already incorporates this overlay. Otherwise the overlay's covered cells
+  //    are excluded from index results for the affected fields and re-evaluated
+  //    against their current values (see the Data Overlay Files specification).
+  //  * Versus other overlays: when two overlays cover the same (offset, field),
+  //    the one with the higher `committed_version` wins. Overlays that share a
+  //    `committed_version` are ordered by their position in
+  //    `DataFragment.overlays`, where a later entry is newer and wins.
+  uint64 committed_version = 3;
+}
+
+// Per-field coverage for a sparse overlay.
+message FieldCoverage {
+  // One entry per field in the overlay's `data_file.fields`, in the same order.
+  // Each is a 32-bit Roaring bitmap of the physical row offsets covered for that
+  // field. An offset present in a field's bitmap but mapped to a NULL value
+  // means the cell is overridden to NULL (distinct from an offset that is absent,
+  // which falls through to the base data file).
+  repeated bytes offset_bitmaps = 1;
+}
+
 // Deletion File
 //
 // The path of the deletion file is constructed as:
@@ -570,14 +642,14 @@ message ShardManifest {
   // files to find actual state.
   uint64 wal_entry_position_last_seen = 4;
 
-  // Next generation ID to create (incremented after each MemTable flush).
+  // Generation to assign to the next SSTable (incremented after each MemTable flush).
   uint64 current_generation = 6;
 
-  // Field 7 removed: merged_generation moved to MemWalIndexDetails.merged_generations
-  // which is the authoritative source for merge progress.
+  // Field 7 removed: compaction progress lives in
+  // MemWalIndexDetails.compacted_sstables.
 
-  // List of flushed MemTable generations and their directory paths.
-  repeated FlushedGeneration flushed_generations = 8;
+  // List of SSTables created by flushing MemTables and their directory paths.
+  repeated SsTable sstables = 8;
 
   // Lifecycle status. Default ACTIVE; SEALED marks an in-flight drop
   // (drop-table 2PC). A SEALED manifest refuses claims at claim_epoch.
@@ -594,40 +666,40 @@ message ShardFieldEntry {
   bytes value = 2;
 }
 
-// A flushed MemTable generation and its storage location.
-message FlushedGeneration {
-  // Generation number.
+// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset.
+message SsTable {
+  // Generation number identifying this SSTable.
   uint64 generation = 1;
 
   // Directory name relative to the shard directory.
   string path = 2;
 }
 
-// A shard's merged generation, used in MemWalIndexDetails.
-message MergedGeneration {
+// A pointer to the latest SSTable compacted for a shard.
+message CompactedSsTable {
   // Shard identifier (UUID v4).
   UUID shard_id = 1;
 
-  // Last generation merged to base table for this shard.
+  // Generation of the latest SSTable compacted into the base table for this shard.
   uint64 generation = 2;
 }
 
-// Tracks which merged generation a base table index has been rebuilt to cover.
-// Used to determine whether to read from flushed MemTable indexes or base table.
+// Tracks which compacted SSTable generation a base table index has been rebuilt to cover.
+// Used to determine whether to read from SSTable indexes or base table.
 message IndexCatchupProgress {
   // Name of the base table index (must match an entry in maintained_indexes).
   string index_name = 1;
 
   // Per-shard progress: the generation up to which this index covers.
   // If a shard is not present, the index is assumed to be fully caught up
-  // (i.e., caught_up_generation >= merged_generation for that shard).
-  repeated MergedGeneration caught_up_generations = 2;
+  // (i.e., caught_up_generation >= compacted_generation for that shard).
+  repeated CompactedSsTable caught_up_generations = 2;
 }
 
 // Index details for MemWAL Index, stored in IndexMetadata.index_details.
 // This is the centralized structure for all MemWAL metadata:
 // - Configuration (sharding specs, indexes to maintain)
-// - Merge progress (merged generations per shard)
+// - SSTable compaction progress
 // - Shard state snapshots
 //
 // Writers read this index to get configuration before writing.
@@ -669,21 +741,21 @@ message MemWalIndexDetails {
   // SQ params) from the base table index to ensure distance comparability.
   repeated string maintained_indexes = 8;
 
-  // Last generation merged to base table for each shard.
+  // Latest SSTable compacted into the base table for each shard.
   // This is updated atomically with merge-insert data commits, enabling
-  // conflict resolution when multiple mergers operate concurrently.
+  // conflict resolution when multiple compactors operate concurrently.
   //
   // Note: This is separate from shard snapshots because:
-  // 1. merged_generations is updated by mergers (atomic with data commit)
+  // 1. compacted_sstables is updated by compactors (atomic with data commit)
   // 2. shard snapshots are updated by background index builder
-  repeated MergedGeneration merged_generations = 9;
+  repeated CompactedSsTable compacted_sstables = 9;
 
   // Per-index catchup progress tracking.
-  // When data is merged to the base table, base table indexes are rebuilt
+  // When data is compacted into the base table, base table indexes are rebuilt
   // asynchronously. This field tracks which generation each index covers.
   //
-  // For indexed queries, if an index's caught_up_generation < merged_generation,
-  // readers should use flushed MemTable indexes for the gap instead of
+  // For indexed queries, if an index's caught_up_generation < compacted_generation,
+  // readers should use SSTable indexes for the gap instead of
   // scanning unindexed data in the base table.
   //
   // If an index is not present in this list, it is assumed to be fully caught up.
diff --git a/protos/transaction.proto b/protos/transaction.proto
index e72e95025a4..c3c5e38190e 100644
--- a/protos/transaction.proto
+++ b/protos/transaction.proto
@@ -243,8 +243,8 @@ message Transaction {
     repeated DataFragment new_fragments = 3;
     // The ids of the fields that have been modified.
     repeated uint32 fields_modified = 4;
-    /// List of MemWAL shard generations to mark as merged after this transaction
-    repeated MergedGeneration merged_generations = 5;
+    /// SSTables to mark as compacted after this transaction.
+    repeated CompactedSsTable compacted_sstables = 5;
     /// The fields that used to judge whether to preserve the new frag's id into
     /// the frag bitmap of the specified indices.
     repeated uint32 fields_for_preserving_frag_bitmap = 6;
@@ -315,12 +315,34 @@ message Transaction {
     repeated DataReplacementGroup replacements = 1;
   }
 
-  // Update the merged generations in MemWAL index.
+  // Overlay files to append to a single fragment, in order (the last entry is
+  // newest). The overlays are appended to the fragment's existing `overlays`
+  // list; they do not replace it, so overlays written by concurrent commits are
+  // preserved.
+  message DataOverlayGroup {
+    uint64 fragment_id = 1;
+    // Each DataOverlayFile.committed_version is left 0 by the writer and stamped
+    // to the new dataset version at commit time (re-stamped on retry), in the
+    // same way as the created-at / last-updated-at version sequences. The fields
+    // touched are read from each overlay's `data_file.fields`.
+    repeated DataOverlayFile overlays = 2;
+  }
+
+  // Attach overlay files to fragments, supplying new values for a subset of
+  // (row offset, field) cells without rewriting the fragments' base data files.
+  // See the DataOverlayFile message in table.proto for resolution, coverage, and
+  // versioning rules, and the Data Overlay Files and Transactions specifications
+  // for the (intentionally permissive) conflict semantics.
+  message DataOverlay {
+    repeated DataOverlayGroup groups = 1;
+  }
+
+  // Update SSTable compaction progress in the MemWAL index.
   // This operation is used during merge-insert to atomically record which
-  // generations have been merged to the base table.
+  // SSTables have been compacted into the base table.
   message UpdateMemWalState {
-    // Shards and generations being marked as merged.
-    repeated MergedGeneration merged_generations = 1;
+    // SSTables being marked as compacted.
+    repeated CompactedSsTable compacted_sstables = 1;
   }
 
   // An operation that updates base paths in the dataset.
@@ -346,6 +368,7 @@ message Transaction {
     UpdateMemWalState update_mem_wal_state = 112;
     Clone clone = 113;
     UpdateBases update_bases = 114;
+    DataOverlay data_overlay = 115;
   }
 
   // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops.
diff --git a/python/Cargo.lock b/python/Cargo.lock
index 440538da1ee..dbaa6aeebad 100644
--- a/python/Cargo.lock
+++ b/python/Cargo.lock
@@ -2,54 +2,6 @@
 # It is not intended for manual editing.
 version = 4
 
-[[package]]
-name = "abi_stable"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445"
-dependencies = [
- "abi_stable_derive",
- "abi_stable_shared",
- "const_panic",
- "core_extensions",
- "crossbeam-channel",
- "generational-arena",
- "libloading",
- "lock_api",
- "parking_lot",
- "paste",
- "repr_offset",
- "rustc_version",
- "serde",
- "serde_derive",
- "serde_json",
-]
-
-[[package]]
-name = "abi_stable_derive"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898"
-dependencies = [
- "abi_stable_shared",
- "as_derive_utils",
- "core_extensions",
- "proc-macro2",
- "quote",
- "rustc_version",
- "syn 1.0.109",
- "typed-arena",
-]
-
-[[package]]
-name = "abi_stable_shared"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63"
-dependencies = [
- "core_extensions",
-]
-
 [[package]]
 name = "adler2"
 version = "2.0.1"
@@ -208,9 +160,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -230,9 +182,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -244,9 +196,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -263,9 +215,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
 dependencies = [
  "bytes",
  "half",
@@ -275,9 +227,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -297,9 +249,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -312,9 +264,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -325,9 +277,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -341,9 +293,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -366,9 +318,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -379,9 +331,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-pyarrow"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce"
+checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec"
 dependencies = [
  "arrow-array",
  "arrow-data",
@@ -391,9 +343,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -404,9 +356,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
 dependencies = [
  "bitflags 2.13.0",
  "serde_core",
@@ -415,9 +367,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -429,9 +381,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -444,18 +396,6 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "as_derive_utils"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4"
-dependencies = [
- "core_extensions",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "async-channel"
 version = "2.5.0"
@@ -485,9 +425,6 @@ name = "async-ffi"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50"
-dependencies = [
- "abi_stable",
-]
 
 [[package]]
 name = "async-lock"
@@ -513,13 +450,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -1188,9 +1125,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
 
 [[package]]
 name = "byteorder"
@@ -1200,9 +1151,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1225,9 +1176,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.65"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1309,9 +1260,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1319,9 +1270,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1331,14 +1282,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -1502,21 +1453,6 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
-[[package]]
-name = "core_extensions"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003"
-dependencies = [
- "core_extensions_proc_macros",
-]
-
-[[package]]
-name = "core_extensions_proc_macros"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea"
-
 [[package]]
 name = "countio"
 version = "0.3.0"
@@ -1583,18 +1519,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1787,14 +1723,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -1821,12 +1756,11 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -1836,9 +1770,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1861,9 +1795,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1884,33 +1818,34 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
  "parquet",
- "paste",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
 dependencies = [
  "futures",
  "log",
@@ -1919,9 +1854,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1941,16 +1876,17 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.4",
+ "parking_lot",
+ "rand 0.9.5",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1972,9 +1908,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1995,9 +1931,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2012,16 +1948,15 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2031,6 +1966,7 @@ dependencies = [
  "datafusion-datasource",
  "datafusion-execution",
  "datafusion-expr",
+ "datafusion-functions",
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-adapter",
@@ -2049,20 +1985,19 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -2071,18 +2006,19 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -2093,35 +2029,33 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-ffi"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c"
+checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e"
 dependencies = [
- "abi_stable",
  "arrow",
  "arrow-schema",
  "async-ffi",
  "async-trait",
+ "chrono",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-datasource",
@@ -2130,22 +2064,25 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
  "datafusion-physical-plan",
  "datafusion-proto",
  "datafusion-proto-common",
  "datafusion-session",
  "futures",
+ "libloading",
  "log",
  "prost",
  "semver",
+ "stabby",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2160,26 +2097,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -2189,19 +2125,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -2210,9 +2145,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2226,34 +2161,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2264,14 +2199,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2279,9 +2213,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
 dependencies = [
  "datafusion-doc",
  "quote",
@@ -2290,9 +2224,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
 dependencies = [
  "arrow",
  "chrono",
@@ -2309,11 +2243,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2321,20 +2254,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2347,26 +2279,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2382,12 +2314,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2402,7 +2335,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2414,9 +2347,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e"
 dependencies = [
  "arrow",
  "chrono",
@@ -2437,14 +2370,13 @@ dependencies = [
  "datafusion-proto-common",
  "object_store",
  "prost",
- "rand 0.9.4",
 ]
 
 [[package]]
 name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2453,9 +2385,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2464,15 +2396,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2484,9 +2415,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2502,9 +2433,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2698,6 +2629,12 @@ dependencies = [
  "encoding_rs",
 ]
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "env_filter"
 version = "2.0.0"
@@ -2850,10 +2787,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -2968,15 +2905,6 @@ dependencies = [
  "cfg-if 0.1.10",
 ]
 
-[[package]]
-name = "generational-arena"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7"
-dependencies = [
- "cfg-if 1.0.4",
-]
-
 [[package]]
 name = "generator"
 version = "0.8.9"
@@ -3087,12 +3015,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -3215,7 +3144,7 @@ dependencies = [
  "hostname",
  "prost",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "serde",
  "thiserror 2.0.18",
@@ -3252,6 +3181,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -3304,6 +3234,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3459,9 +3394,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
@@ -3867,18 +3802,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82"
+checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.10.1"
+version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9"
+checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d"
 dependencies = [
  "bytecount",
  "cedarwood",
@@ -4015,7 +3950,7 @@ dependencies = [
  "nom",
  "num-traits",
  "ordered-float 5.3.0",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "serde_json",
  "zmij",
@@ -4049,7 +3984,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4105,7 +4040,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
  "roaring",
  "rustc-hash",
@@ -4123,7 +4058,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4132,13 +4067,14 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -4165,7 +4101,7 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4175,7 +4111,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4198,7 +4134,7 @@ dependencies = [
  "object_store",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "roaring",
  "serde_json",
  "snafu",
@@ -4213,7 +4149,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4245,7 +4181,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4255,14 +4191,14 @@ dependencies = [
  "futures",
  "half",
  "hex",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rand_xoshiro",
 ]
 
 [[package]]
 name = "lance-derive"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4271,7 +4207,7 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4296,7 +4232,7 @@ dependencies = [
  "num-traits",
  "prost",
  "prost-build",
- "rand 0.9.4",
+ "rand 0.9.5",
  "strum 0.26.3",
  "tokio",
  "tracing",
@@ -4306,7 +4242,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4336,7 +4272,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4350,12 +4286,13 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arc-swap",
  "arrow",
  "arrow-arith",
  "arrow-array",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -4389,6 +4326,7 @@ dependencies = [
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
@@ -4402,7 +4340,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rangemap",
  "rayon",
@@ -4417,9 +4355,31 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "10.0.0-beta.3"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4437,12 +4397,14 @@ dependencies = [
  "bytes",
  "chrono",
  "futures",
+ "goosefs-sdk",
  "http 1.4.2",
  "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
  "log",
+ "metrics",
  "moka",
  "object_store",
  "object_store_opendal",
@@ -4450,7 +4412,7 @@ dependencies = [
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "tempfile",
  "tokio",
@@ -4460,7 +4422,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4470,13 +4432,13 @@ dependencies = [
  "lance-arrow",
  "lance-core",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4488,7 +4450,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4508,7 +4470,7 @@ dependencies = [
  "lance-table",
  "log",
  "object_store",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "roaring",
  "serde",
@@ -4537,7 +4499,7 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4552,7 +4514,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4576,7 +4538,7 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rangemap",
  "roaring",
  "semver",
@@ -4591,7 +4553,7 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",
@@ -4670,18 +4632,18 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libloading"
-version = "0.7.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
 dependencies = [
  "cfg-if 1.0.4",
- "winapi",
+ "windows-link",
 ]
 
 [[package]]
@@ -4920,6 +4882,36 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.24.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
+dependencies = [
+ "portable-atomic",
+ "rapidhash",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
+dependencies = [
+ "aho-corasick",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.5",
+ "indexmap 2.14.0",
+ "metrics",
+ "ordered-float 4.6.0",
+ "quanta",
+ "radix_trie",
+ "rand 0.9.5",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -5024,6 +5016,15 @@ dependencies = [
  "rawpointer",
 ]
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "nom"
 version = "8.0.0"
@@ -5547,6 +5548,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-float"
 version = "5.3.0"
@@ -5612,9 +5622,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "58.3.0"
+version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -6029,7 +6039,7 @@ dependencies = [
 
 [[package]]
 name = "pylance"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 dependencies = [
  "alloc-stdlib",
  "arrow",
@@ -6061,6 +6071,8 @@ dependencies = [
  "lance-table",
  "libc",
  "log",
+ "metrics",
+ "metrics-util",
  "object_store",
  "prost",
  "prost-types",
@@ -6146,6 +6158,21 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -6196,7 +6223,7 @@ dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.9.5",
  "ring",
  "rustc-hash",
  "rustls",
@@ -6249,6 +6276,16 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rancor"
 version = "0.1.1"
@@ -6270,9 +6307,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -6340,7 +6377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -6358,6 +6395,24 @@ version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
 
+[[package]]
+name = "rapidhash"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "11.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+dependencies = [
+ "bitflags 2.13.0",
+]
+
 [[package]]
 name = "rawpointer"
 version = "0.2.1"
@@ -6487,15 +6542,6 @@ dependencies = [
  "bytecheck",
 ]
 
-[[package]]
-name = "repr_offset"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea"
-dependencies = [
- "tstr",
-]
-
 [[package]]
 name = "reqsign-aliyun-oss"
 version = "3.1.0"
@@ -6852,9 +6898,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -7111,9 +7157,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -7121,22 +7167,22 @@ dependencies = [
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.118",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -7152,10 +7198,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -7310,6 +7357,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "sha2-const-stable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9"
+
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -7384,6 +7437,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -7398,18 +7457,18 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522"
+checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7"
 dependencies = [
  "snafu-derive",
 ]
 
 [[package]]
 name = "snafu-derive"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda"
+checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -7463,9 +7522,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -7482,6 +7541,40 @@ dependencies = [
  "syn 2.0.118",
 ]
 
+[[package]]
+name = "stabby"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec"
+dependencies = [
+ "rustversion",
+ "stabby-abi",
+]
+
+[[package]]
+name = "stabby-abi"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4"
+dependencies = [
+ "rustc_version",
+ "rustversion",
+ "sha2-const-stable",
+ "stabby-macros",
+]
+
+[[package]]
+name = "stabby-macros"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.1"
@@ -7576,11 +7669,12 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -7613,9 +7707,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
+version = "2.0.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7624,9 +7718,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.118"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7856,9 +7950,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.52.3"
+version = "1.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
 dependencies = [
  "bytes",
  "libc",
@@ -7905,9 +7999,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -7917,13 +8011,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
+ "libc",
  "pin-project-lite",
  "tokio",
 ]
@@ -8173,36 +8268,15 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "tstr"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7"
-dependencies = [
- "tstr_proc_macros",
-]
-
-[[package]]
-name = "tstr_proc_macros"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a"
-
 [[package]]
 name = "twox-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
 dependencies = [
- "rand 0.9.4",
+ "rand 0.10.1",
 ]
 
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
-
 [[package]]
 name = "typenum"
 version = "1.20.1"
@@ -8351,9 +8425,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.4"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
  "getrandom 0.4.3",
  "js-sys",
@@ -9081,9 +9155,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.16"
+version = "0.8.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
+checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
 
 [[package]]
 name = "yoke"
diff --git a/python/Cargo.toml b/python/Cargo.toml
index f0c84c01a16..9df1ca7c21e 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "9.0.0-beta.16"
+version = "10.0.0-beta.3"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -25,9 +25,9 @@ arrow-cast = "58.0.0"
 arrow-data = "58.0.0"
 arrow-schema = "58.0.0"
 object_store = "0.13.2"
-datafusion = { version = "53.0.0", default-features = false }
-datafusion-ffi = "53.0.0"
-datafusion-common = "53.0.0"
+datafusion = { version = "54.0.0", default-features = false }
+datafusion-ffi = "54.0.0"
+datafusion-common = "54.0.0"
 # Keep the Python FFI build on the working Brotli allocator resolution until
 # datafusion-ffi no longer enables datafusion-proto/default.
 # See https://github.com/lance-format/lance/issues/7271.
@@ -44,6 +44,7 @@ lance = { path = "../rust/lance", features = [
     "goosefs",
     "dynamodb",
     "substrait",
+    "metrics",
 ] }
 lance-arrow = { path = "../rust/lance-arrow" }
 lance-core = { path = "../rust/lance-core" }
@@ -54,7 +55,7 @@ lance-index = { path = "../rust/lance-index", features = [
     "tokenizer-lindera",
     "tokenizer-jieba",
 ] }
-lance-io = { path = "../rust/lance-io" }
+lance-io = { path = "../rust/lance-io", features = ["metrics"] }
 lance-linalg = { path = "../rust/lance-linalg" }
 lance-namespace = { path = "../rust/lance-namespace" }
 lance-namespace-impls = { path = "../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] }
@@ -62,6 +63,8 @@ lance-table = { path = "../rust/lance-table" }
 lance-datafusion = { path = "../rust/lance-datafusion" }
 libc = "0.2.176"
 log = "0.4"
+metrics = "0.24"
+metrics-util = "0.19"
 prost = "0.14.1"
 prost-types = "0.14.1"
 pyo3 = { version = "0.28", features = [
@@ -80,7 +83,7 @@ serde_yaml = "0.9.34"
 tracing-chrome = "0.7.1"
 tracing-subscriber = "0.3.17"
 tracing = { version = "0.1" }
-bytes = "1.4"
+bytes = "1.11.1"
 
 [features]
 default = []
diff --git a/python/pyproject.toml b/python/pyproject.toml
index dd11344324c..d89d3671dca 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -49,21 +49,20 @@ build-backend = "maturin"
 [project.optional-dependencies]
 tests = [
     "boto3",
-    "datasets",
-    "duckdb",
+    "datasets==4.4.0",
+    "duckdb>=1.5.0,<1.6.0",
     "ml_dtypes",
     "pillow",
     "pandas",
     "polars[pyarrow,pandas]",
     "psutil",
     "pytest",
-    # Only test tensorflow on linux for now. We will deprecate tensorflow soon.
-    "tensorflow; sys_platform == 'linux'",
     "tqdm",
-    "datafusion>=53,<54",
+    "datafusion>=54,<55",
 ]
 dev = ["ruff==0.11.2", "pyright"]
 benchmarks = ["pytest-benchmark"]
+otel = ["opentelemetry-api", "opentelemetry-sdk"]
 torch = ["torch>=2.0"]
 geo = [
     "geoarrow-rust-core",
@@ -73,17 +72,17 @@ geo = [
 [dependency-groups]
 tests = [
     "boto3==1.40.43",
-    "datasets==4.1.1",
-    "duckdb==1.4.0",
+    "datasets==4.4.0",
+    "duckdb>=1.5.0,<1.6.0",
     "ml_dtypes==0.5.3",
     "pillow==11.3.0",
     "pandas==2.3.3",
     "polars[pyarrow,pandas]==1.34.0",
     "psutil==7.1.0",
     "pytest==8.4.2",
-    "tensorflow==2.20.0; sys_platform == 'linux'",
     "tqdm==4.67.1",
-    "datafusion==53.0.0",
+    "datafusion==54.0.0",
+    "opentelemetry-sdk==1.30.0",
 ]
 dev = [
     "maturin==1.13.3",
@@ -104,7 +103,7 @@ lint.select = ["F", "E", "W", "I", "G", "TCH", "PERF", "B019"]
 "*.pyi" = ["E301", "E302"]
 
 [tool.pyright]
-pythonVersion = "3.13"
+pythonVersion = "3.14"
 # TODO: expand this list as we fix more files.
 include = [
     "python/lance/util.py",
@@ -137,9 +136,6 @@ markers = [
 filterwarnings = [
     'error::FutureWarning',
     'error::DeprecationWarning',
-    # TensorFlow import can emit NumPy deprecation FutureWarnings in some environments.
-    # We keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning.
-    'ignore:.*`np\\.object` will be defined as the corresponding NumPy scalar\\..*:FutureWarning',
     # Boto3
     'ignore:.*datetime\.datetime\.utcnow\(\) is deprecated.*:DeprecationWarning',
     # Hugging Face Hub calls this deprecated hf-xet API internally.
@@ -153,9 +149,8 @@ filterwarnings = [
     'ignore:.*the load_module\(\) method is deprecated.*:DeprecationWarning',
     # Pytorch uses deprecated jit.script_method internally (torch/utils/mkldnn.py)
     'ignore:.*torch\.jit\.script_method.*is deprecated.*:DeprecationWarning',
+    # Pytorch uses the same API internally on Python 3.14+ with a different warning message.
+    'ignore:.*torch\.jit\.script_method.*is not supported in Python 3\.14\+.*:DeprecationWarning',
     # huggingface_hub still calls the deprecated hf_xet.download_files() during Xet downloads
     'ignore:.*hf_xet\.download_files\(\) is deprecated.*:DeprecationWarning',
-    # TensorFlow/Keras import can emit NumPy deprecation FutureWarnings in some environments.
-    # Keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning.
-    'ignore:.*np\.object.*:FutureWarning',
 ]
diff --git a/python/python/benchmarks/test_blob.py b/python/python/benchmarks/test_blob.py
new file mode 100644
index 00000000000..c2465f36fad
--- /dev/null
+++ b/python/python/benchmarks/test_blob.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+from itertools import count
+from time import process_time
+
+import pyarrow as pa
+import pytest
+from lance.file import LanceFileSession
+
+# Many small blobs isolate per-row Python overhead; fewer large blobs show how
+# the bulk path behaves once payload copying accounts for more of the CPU time.
+WORKLOADS = [
+    pytest.param(50_000, 256, id="50000x256b"),
+    pytest.param(2_000, 64 * 1024, id="2000x64kib"),
+]
+
+
+def _packed_blob_benchmark(benchmark, tmpdir_factory, row_count, payload_size, mode):
+    payload = b"x" * payload_size
+    payloads = pa.repeat(payload, row_count)
+    files = LanceFileSession(str(tmpdir_factory.mktemp("packed_blob_writer")))
+    file_number = count()
+
+    if mode == "scalar_preconverted":
+        python_payloads = payloads.to_pylist()
+
+        def write():
+            writer = files.open_packed_blob_writer(
+                f"scalar-{next(file_number)}.lance", 1
+            )
+            for value in python_payloads:
+                writer.write_blob(value)
+            return writer.finish()
+
+    elif mode == "scalar_from_arrow":
+
+        def write():
+            writer = files.open_packed_blob_writer(
+                f"scalar-arrow-{next(file_number)}.lance", 1
+            )
+            for value in payloads.to_pylist():
+                writer.write_blob(value)
+            return writer.finish()
+
+    elif mode == "bulk":
+
+        def write():
+            writer = files.open_packed_blob_writer(f"bulk-{next(file_number)}.lance", 1)
+            writer.write_blobs(payloads)
+            return writer.finish_array("blob")
+
+    else:
+        raise ValueError(f"Unknown benchmark mode: {mode}")
+
+    result = benchmark.pedantic(write, iterations=1, rounds=5)
+    assert len(result) == row_count
+
+
+@pytest.mark.benchmark(group="packed_blob_writer_cpu", timer=process_time)
+@pytest.mark.parametrize("row_count,payload_size", WORKLOADS)
+@pytest.mark.parametrize(
+    "mode",
+    ["scalar_preconverted", "scalar_from_arrow", "bulk"],
+)
+def test_packed_blob_writer(benchmark, tmpdir_factory, row_count, payload_size, mode):
+    _packed_blob_benchmark(
+        benchmark,
+        tmpdir_factory,
+        row_count,
+        payload_size,
+        mode,
+    )
diff --git a/python/python/benchmarks/test_search.py b/python/python/benchmarks/test_search.py
index b4e33338cb1..b86a2818e97 100644
--- a/python/python/benchmarks/test_search.py
+++ b/python/python/benchmarks/test_search.py
@@ -210,6 +210,49 @@ def test_ann_with_refine(test_dataset, benchmark):
     assert result.num_rows > 0
 
 
+N_BATCH_QUERIES = 32
+
+
+@pytest.mark.benchmark(group="query_ann_batch")
+def test_batch_ann_search(test_dataset, benchmark):
+    # One request carrying all query vectors: the index shares each partition's
+    # scan across the batch (issue #6822).
+    queries = np.random.randn(N_BATCH_QUERIES, N_DIMS).astype(np.float32)
+    result = benchmark(
+        test_dataset.to_table,
+        columns=[],
+        with_row_id=True,
+        nearest=dict(
+            column="vector",
+            q=queries,
+            k=100,
+            nprobes=10,
+        ),
+    )
+    assert result.num_rows > 0
+
+
+@pytest.mark.benchmark(group="query_ann_batch")
+def test_repeated_single_ann_search(test_dataset, benchmark):
+    # Baseline: the same query vectors issued one indexed search at a time.
+    queries = np.random.randn(N_BATCH_QUERIES, N_DIMS).astype(np.float32)
+
+    def run():
+        for q in queries:
+            test_dataset.to_table(
+                columns=[],
+                with_row_id=True,
+                nearest=dict(
+                    column="vector",
+                    q=q,
+                    k=100,
+                    nprobes=10,
+                ),
+            )
+
+    benchmark(run)
+
+
 @pytest.mark.benchmark(group="query_ann")
 @pytest.mark.parametrize("selectivity", (0.25, 0.75))
 @pytest.mark.parametrize("prefilter", (False, True))
diff --git a/python/python/benchmarks/zone_map_seeds.py b/python/python/benchmarks/zone_map_seeds.py
new file mode 100644
index 00000000000..023e14cfcf7
--- /dev/null
+++ b/python/python/benchmarks/zone_map_seeds.py
@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+"""
+Benchmark: Zone Map Index Seeds vs. No Seeds
+
+Measures the time to update a zone map index when new data is appended to an
+already-indexed dataset, with and without use_seeds.
+
+Seeds allow Lance to embed per-zone min/max statistics in data files at write
+time.  When optimize_indices() runs, it harvests those pre-computed stats
+instead of re-scanning the full column, which eliminates the read I/O for
+wide data types.
+
+WHY the workflow matters
+------------------------
+Seeds only engage when merging new unindexed fragments into an existing
+non-empty index segment.  A first optimize_indices() call on a freshly
+created empty-index always does a full rebuild (no old fragments to merge
+against), so seeds would never be used.
+
+Correct workflow:
+  1. Write an initial batch of data and build the zone map index on it.
+     This creates a non-empty indexed segment; all subsequent writes will
+     embed seed buffers in their data files (when use_seeds=True).
+  2. Append the bulk of the data in multiple batches.
+  3. optimize_indices() — harvests seeds (phase 2 data) and merges them into
+     the segment from phase 1.  Without seeds, phase 3 must re-read all of
+     the phase-2 data from disk.
+
+Phases timed:
+  initial_write   write seed_fraction of total rows + create/build index
+  ingest          append remaining rows (seeds collected here when enabled)
+  update_index    optimize_indices() — seed harvest happens here
+  total           sum of the three phases above
+
+Three scenarios:
+  int64                    10 M rows   ~80 MB on disk
+  FixedSizeBinary(4096)    10 M rows   ~38 GB on disk
+  LargeBinary(20 KiB)       1 M rows   ~19 GB on disk
+
+Usage:
+    uv run python python/benchmarks/zone_map_seeds.py
+    uv run python python/benchmarks/zone_map_seeds.py --tmpdir /fast/nvme/bench
+    uv run python python/benchmarks/zone_map_seeds.py --scale 0.01
+"""
+
+import argparse
+import os
+import shutil
+import tempfile
+import time
+from pathlib import Path
+
+import lance
+import numpy as np
+import pyarrow as pa
+from lance.indices import IndexConfig
+
+KiB = 1024
+
+# Fraction of total rows written in the initial indexed batch.
+SEED_FRACTION = 0.1
+
+
+# ---------------------------------------------------------------------------
+# Data generators (yield pa.Table one batch at a time)
+# ---------------------------------------------------------------------------
+
+
+def gen_int_batches(num_rows: int, batch_size: int):
+    rng = np.random.default_rng()
+    for start in range(0, num_rows, batch_size):
+        n = min(batch_size, num_rows - start)
+        yield pa.table({"value": pa.array(rng.integers(0, 2**31, n, dtype=np.int64))})
+
+
+def gen_fsb_batches(num_rows: int, blob_bytes: int, batch_size: int):
+    for start in range(0, num_rows, batch_size):
+        n = min(batch_size, num_rows - start)
+        raw = os.urandom(n * blob_bytes)
+        arr = pa.FixedSizeBinaryArray.from_buffers(
+            pa.binary(blob_bytes), n, [None, pa.py_buffer(raw)]
+        )
+        yield pa.table({"value": arr})
+
+
+def gen_large_binary_batches(num_rows: int, blob_bytes: int, batch_size: int):
+    for start in range(0, num_rows, batch_size):
+        n = min(batch_size, num_rows - start)
+        yield pa.table(
+            {
+                "value": pa.array(
+                    [os.urandom(blob_bytes) for _ in range(n)],
+                    type=pa.large_binary(),
+                )
+            }
+        )
+
+
+# ---------------------------------------------------------------------------
+# Core benchmark runner
+# ---------------------------------------------------------------------------
+
+
+def run_scenario(
+    dataset_path: Path,
+    schema: pa.Schema,
+    seed_rows: int,
+    seed_batch_size: int,
+    gen_seed_batches,  # () -> Iterator[pa.Table] for the initial write
+    gen_bulk_batches,  # () -> Iterator[pa.Table] for the appended data
+    use_seeds: bool,
+) -> dict:
+    """
+    Run one full scenario and return per-phase timings.
+
+    Seeds are only engaged by optimize_indices() when there is at least one
+    previously indexed fragment.  This scenario ensures that by writing
+    seed_rows of data and building the index before any bulk ingest.
+    """
+    if dataset_path.exists():
+        shutil.rmtree(dataset_path)
+
+    index_config = IndexConfig(
+        index_type="ZONEMAP", parameters={"use_seeds": use_seeds}
+    )
+    timings = {}
+
+    # Phase 1: initial write + index build
+    t0 = time.perf_counter()
+    ds = None
+    for batch in gen_seed_batches():
+        if ds is None:
+            ds = lance.write_dataset(batch, dataset_path, schema=schema)
+        else:
+            ds = lance.write_dataset(batch, dataset_path, mode="append")
+    ds.create_scalar_index("value", index_type=index_config, replace=True)
+    timings["initial_write"] = time.perf_counter() - t0
+
+    # Phase 2: bulk ingest — seeds are collected per-batch when enabled
+    t0 = time.perf_counter()
+    for batch in gen_bulk_batches():
+        ds = lance.write_dataset(batch, dataset_path, mode="append")
+    timings["ingest"] = time.perf_counter() - t0
+
+    # Phase 3: index update — seed harvest happens here
+    t0 = time.perf_counter()
+    ds.optimize.optimize_indices()
+    timings["update_index"] = time.perf_counter() - t0
+
+    timings["total"] = sum(timings.values())
+    shutil.rmtree(dataset_path)
+    return timings
+
+
+# ---------------------------------------------------------------------------
+# Output formatting
+# ---------------------------------------------------------------------------
+
+
+def print_comparison(title: str, no_seeds: dict, with_seeds: dict) -> None:
+    phases = ["initial_write", "ingest", "update_index", "total"]
+    col_w = 13
+
+    print(f"\n{'=' * 70}")
+    print(f"  {title}")
+    print(f"{'=' * 70}")
+    print(
+        f"  {'Phase':<20} {'No Seeds':>{col_w}} "
+        + f"{'With Seeds':>{col_w}} {'Speedup':>{col_w}}"
+    )
+    print(f"  {'-' * (20 + col_w * 3 + 6)}")
+    for phase in phases:
+        t_no = no_seeds[phase]
+        t_yes = with_seeds[phase]
+        speedup = t_no / t_yes if t_yes > 0 else float("inf")
+        marker = "* " if phase == "total" else "  "
+        print(
+            f"{marker}{phase:<20} {t_no:>{col_w}.2f}s"
+            f" {t_yes:>{col_w}.2f}s"
+            f" {speedup:>{col_w - 1}.2f}x"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Per-type benchmark wrappers
+# ---------------------------------------------------------------------------
+
+
+def bench_integers(base: Path, num_rows: int) -> None:
+    seed_rows = max(1, int(num_rows * SEED_FRACTION))
+    bulk_rows = num_rows - seed_rows
+    # ~400 KB per seed batch; ~4 MB per bulk batch → ~20 bulk fragments
+    seed_bs = max(1, seed_rows // 5)
+    bulk_bs = max(1, bulk_rows // 20)
+    print(
+        f"\nInteger (int64): {num_rows:,} rows "
+        f"(initial={seed_rows:,}, bulk={bulk_rows:,}) ..."
+    )
+    schema = pa.schema([pa.field("value", pa.int64())])
+
+    no_seeds = run_scenario(
+        base / "int_no_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_int_batches(seed_rows, seed_bs),
+        lambda: gen_int_batches(bulk_rows, bulk_bs),
+        use_seeds=False,
+    )
+    with_seeds = run_scenario(
+        base / "int_with_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_int_batches(seed_rows, seed_bs),
+        lambda: gen_int_batches(bulk_rows, bulk_bs),
+        use_seeds=True,
+    )
+    print_comparison(f"int64  —  {num_rows:,} rows", no_seeds, with_seeds)
+
+
+def bench_vectors(base: Path, num_rows: int) -> None:
+    blob_bytes = 4 * KiB
+    seed_rows = max(1, int(num_rows * SEED_FRACTION))
+    bulk_rows = num_rows - seed_rows
+    total_gb = num_rows * blob_bytes / (1024**3)
+    # ~200 MB per batch to stay memory-friendly
+    seed_bs = max(1, (200 * KiB * KiB) // blob_bytes)
+    bulk_bs = seed_bs
+    print(
+        f"\nVector (FixedSizeBinary {blob_bytes // KiB} KiB): {num_rows:,} rows"
+        f"  (~{total_gb:.1f} GB)  (initial={seed_rows:,}, bulk={bulk_rows:,}) ..."
+    )
+    schema = pa.schema([pa.field("value", pa.binary(blob_bytes))])
+
+    no_seeds = run_scenario(
+        base / "vec_no_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs),
+        lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs),
+        use_seeds=False,
+    )
+    with_seeds = run_scenario(
+        base / "vec_with_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs),
+        lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs),
+        use_seeds=True,
+    )
+    print_comparison(
+        f"FixedSizeBinary({blob_bytes // KiB} KiB)  —  {num_rows:,} rows",
+        no_seeds,
+        with_seeds,
+    )
+
+
+def bench_large_binary(base: Path, num_rows: int) -> None:
+    blob_bytes = 20 * KiB
+    seed_rows = max(1, int(num_rows * SEED_FRACTION))
+    bulk_rows = num_rows - seed_rows
+    total_gb = num_rows * blob_bytes / (1024**3)
+    # ~200 MB per batch
+    seed_bs = max(1, (200 * KiB * KiB) // blob_bytes)
+    bulk_bs = seed_bs
+    print(
+        f"\nLargeBinary ({blob_bytes // KiB} KiB blobs): {num_rows:,} rows"
+        f"  (~{total_gb:.1f} GB)  (initial={seed_rows:,}, bulk={bulk_rows:,}) ..."
+    )
+    schema = pa.schema([pa.field("value", pa.large_binary())])
+
+    no_seeds = run_scenario(
+        base / "bin_no_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs),
+        lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs),
+        use_seeds=False,
+    )
+    with_seeds = run_scenario(
+        base / "bin_with_seeds",
+        schema,
+        seed_rows,
+        seed_bs,
+        lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs),
+        lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs),
+        use_seeds=True,
+    )
+    print_comparison(
+        f"LargeBinary({blob_bytes // KiB} KiB)  —  {num_rows:,} rows",
+        no_seeds,
+        with_seeds,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--tmpdir",
+        type=Path,
+        default=None,
+        help="Directory for benchmark datasets (default: system temp dir)",
+    )
+    parser.add_argument(
+        "--scale",
+        type=float,
+        default=1.0,
+        help="Row-count scale factor (default 1.0; use 0.01 for a quick smoke test)",
+    )
+    args = parser.parse_args()
+
+    int_rows = max(2, int(10_000_000 * args.scale))
+    vec_rows = max(2, int(10_000_000 * args.scale))
+    bin_rows = max(2, int(1_000_000 * args.scale))
+
+    print(
+        f"Scale: {args.scale}  →  "
+        f"{int_rows:,} int / {vec_rows:,} vector / {bin_rows:,} binary rows"
+    )
+
+    def run(tmp_dir: Path) -> None:
+        bench_integers(tmp_dir, int_rows)
+        bench_vectors(tmp_dir, vec_rows)
+        bench_large_binary(tmp_dir, bin_rows)
+
+    if args.tmpdir is not None:
+        args.tmpdir.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory(
+            prefix="lance_zonemap_bench_", dir=args.tmpdir
+        ) as tmp:
+            run(Path(tmp))
+    else:
+        with tempfile.TemporaryDirectory(prefix="lance_zonemap_bench_") as tmp:
+            run(Path(tmp))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/python/python/lance/__init__.py b/python/python/lance/__init__.py
index 61d94b5550a..33f4ce00344 100644
--- a/python/python/lance/__init__.py
+++ b/python/python/lance/__init__.py
@@ -47,13 +47,14 @@
     ScanStatistics,
     bytes_read_counter,
     iops_counter,
+    simd_info,
 )
 from .mem_wal import (
+    CompactedSsTable,
     ExecutionPlan,
     LsmPointLookupPlanner,
     LsmScanner,
     LsmVectorSearchPlanner,
-    MergedGeneration,
     ShardingField,
     ShardingSpec,
     ShardSnapshot,
@@ -115,6 +116,7 @@
     "json_to_schema",
     "schema_to_json",
     "set_logger",
+    "simd_info",
     "write_dataset",
     "FFILanceTableProvider",
     "IndexProgress",
@@ -122,7 +124,7 @@
     "LsmPointLookupPlanner",
     "LsmScanner",
     "LsmVectorSearchPlanner",
-    "MergedGeneration",
+    "CompactedSsTable",
     "ShardSnapshot",
     "ShardWriter",
     "ShardingField",
diff --git a/python/python/lance/arrow.py b/python/python/lance/arrow.py
index da022cdc8f6..5e9f671ab80 100644
--- a/python/python/lance/arrow.py
+++ b/python/python/lance/arrow.py
@@ -310,16 +310,7 @@ def pillow_metadata_decoder(images):
             img = Image.open(io.BytesIO(images[0].as_py()))
             return img
 
-        def tensorflow_metadata_decoder(images):
-            import tensorflow as tf
-
-            img = tf.io.decode_image(images[0].as_py())
-            return img
-
-        decoders = (
-            ("tensorflow", tensorflow_metadata_decoder),
-            ("PIL", pillow_metadata_decoder),
-        )
+        decoders = (("PIL", pillow_metadata_decoder),)
         decoder = None
 
         for libname, metadata_decoder in decoders:
@@ -351,7 +342,7 @@ def to_tensor(
         decoder : Callable[pa.binary()], optional
             A function that takes a binary array and returns a numpy.ndarray
             or pa.fixed_shape_tensor. If not provided, will attempt to use
-            tensorflow and then pillow decoder in that order.
+            pillow.
 
         Returns
         -------
@@ -385,20 +376,7 @@ def pillow_decoder(images) -> "np.ndarray":
                     ]
                 )
 
-            def tensorflow_decoder(images) -> "np.ndarray":
-                import tensorflow as tf
-
-                decoded_to_tensor = tuple(
-                    tf.io.decode_image(img) for img in images.to_pylist()
-                )
-                return tf.stack(  # pyright: ignore[reportOptionalCall]
-                    decoded_to_tensor, axis=0
-                ).numpy()
-
-            decoders = [
-                ("tensorflow", tensorflow_decoder),
-                ("PIL", pillow_decoder),
-            ]
+            decoders = [("PIL", pillow_decoder)]
             for libname, decoder_function in decoders:
                 try:
                     __import__(libname)
@@ -408,8 +386,8 @@ def tensorflow_decoder(images) -> "np.ndarray":
                     pass
             else:
                 raise ValueError(
-                    "No image decoder available. Please either install one of "
-                    "tensorflow, pillow, or pass a decoder argument."
+                    "No image decoder available. Please install pillow or pass a "
+                    "decoder argument."
                 )
 
         image_array = decoder(self.storage)
@@ -499,19 +477,8 @@ def pillow_encoder(x):
                     encoded_images.append(buf.getvalue())
             return pa.array(encoded_images, type=storage_type)
 
-        def tensorflow_encoder(x):
-            import tensorflow as tf
-
-            encoded_images = (
-                tf.io.encode_png(y).numpy() for y in tf.convert_to_tensor(x)
-            )
-            return pa.array(encoded_images, type=storage_type)
-
         if not encoder:
-            encoders = (
-                ("PIL", pillow_encoder),
-                ("tensorflow", tensorflow_encoder),
-            )
+            encoders = (("PIL", pillow_encoder),)
             for libname, encoder_function in encoders:
                 try:
                     __import__(libname)
@@ -521,8 +488,8 @@ def tensorflow_encoder(x):
                     pass
             else:
                 raise ValueError(
-                    "No image encoder available. Please either install one of "
-                    "tensorflow, pillow, or pass an encoder argument."
+                    "No image encoder available. Please install pillow or pass an "
+                    "encoder argument."
                 )
 
         return EncodedImageArray.from_storage(
diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py
index 6be0a78d2e8..e38acfd7a07 100644
--- a/python/python/lance/dataset.py
+++ b/python/python/lance/dataset.py
@@ -71,11 +71,15 @@
 from .lance import __version__ as __version__
 from .lance import _Session as Session
 from .query import FullTextQuery
-from .types import _coerce_reader
+from .types import _coerce_reader, _is_materialized
 from .udf import BatchUDF, normalize_transform
 from .udf import BatchUDFCheckpoint as BatchUDFCheckpoint
 from .udf import batch_udf as batch_udf
-from .util import _target_partition_size_to_num_partitions, td_to_micros
+from .util import (
+    _normalize_index_segment_ids,
+    _target_partition_size_to_num_partitions,
+    td_to_micros,
+)
 
 if TYPE_CHECKING:
     from pyarrow._compute import Expression
@@ -393,6 +397,12 @@ def execute(self, data_obj: ReaderLike, *, schema: Optional[pa.Schema] = None):
         """
         reader = _coerce_reader(data_obj, schema)
 
+        # Materialized sources are wrapped in an in-memory table so retries never
+        # spill and the source's statistics can drive the join; everything else is
+        # treated as a one-shot stream.
+        if _is_materialized(data_obj):
+            return super(MergeInsertBuilder, self).execute_batches(reader)
+
         return super(MergeInsertBuilder, self).execute(reader)
 
     def execute_uncommitted(
@@ -417,6 +427,9 @@ def execute_uncommitted(
         """
         reader = _coerce_reader(data_obj, schema)
 
+        if _is_materialized(data_obj):
+            return super(MergeInsertBuilder, self).execute_uncommitted_batches(reader)
+
         return super(MergeInsertBuilder, self).execute_uncommitted(reader)
 
     # These next three overrides exist only to document the methods
@@ -735,23 +748,22 @@ def analyze_plan(
         reader = _coerce_reader(data_obj, schema)
         return super(MergeInsertBuilder, self).analyze_plan(reader)
 
-    def mark_generations_as_merged(
-        self, generations: "List[mem_wal.MergedGeneration]"
+    def mark_sstables_as_compacted(
+        self, sstables: "List[mem_wal.CompactedSsTable]"
     ) -> "MergeInsertBuilder":
-        """Mark MemWAL generations as merged into the base table.
+        """Mark MemWAL SSTables as compacted into the base table.
 
-        Call this before executing the merge_insert when the source data
-        includes rows from MemWAL flushed generations.
+        Call this before executing merge_insert when it compacts MemWAL SSTables.
 
         Parameters
         ----------
-        generations : list of MergedGeneration
-            Generations to mark as merged.
+        sstables : list of CompactedSsTable
+            SSTables to mark as compacted.
         """
-        from .mem_wal import _to_raw_merged_generations
+        from .mem_wal import _to_raw_compacted_sstables
 
-        raw_gens = _to_raw_merged_generations(generations)
-        super(MergeInsertBuilder, self).mark_generations_as_merged(raw_gens)
+        raw_sstables = _to_raw_compacted_sstables(sstables)
+        super(MergeInsertBuilder, self).mark_sstables_as_compacted(raw_sstables)
         return self
 
 
@@ -2162,7 +2174,7 @@ def take_blobs(
         ids: Optional[Union[List[int], pa.Array]] = None,
         addresses: Optional[Union[List[int], pa.Array]] = None,
         indices: Optional[Union[List[int], pa.Array]] = None,
-    ) -> List[BlobFile]:
+    ) -> List[Optional[BlobFile]]:
         """
         Select blobs by row IDs.
 
@@ -2189,7 +2201,9 @@ def take_blobs(
 
         Returns
         -------
-        blob_files : List[BlobFile]
+        blob_files : List[Optional[BlobFile]]
+            One element per selected row. Null blob values return ``None``;
+            valid empty blobs return a ``BlobFile`` with size zero.
         """
         selection_kind, selection_values = _resolve_blob_selection(
             ids, addresses, indices
@@ -2205,7 +2219,10 @@ def take_blobs(
             lance_blob_files = self._ds.take_blobs_by_indices(
                 selection_values, blob_column
             )
-        return [BlobFile(lance_blob_file) for lance_blob_file in lance_blob_files]
+        return [
+            BlobFile(lance_blob_file) if lance_blob_file is not None else None
+            for lance_blob_file in lance_blob_files
+        ]
 
     def read_blobs(
         self,
@@ -2216,7 +2233,7 @@ def read_blobs(
         *,
         io_buffer_size: Optional[int] = None,
         preserve_order: Optional[bool] = None,
-    ) -> List[Tuple[int, bytes]]:
+    ) -> List[Tuple[int, Optional[bytes]]]:
         """
         Read blobs directly into memory using Lance's planned blob reader.
 
@@ -2244,8 +2261,9 @@ def read_blobs(
 
         Returns
         -------
-        blobs : List[Tuple[int, bytes]]
-            A list of ``(row_address, blob_bytes)`` pairs.
+        blobs : List[Tuple[int, Optional[bytes]]]
+            One ``(row_address, blob_bytes)`` pair per selected row. Null blob
+            values return ``None``; valid empty blobs return ``b""``.
         """
         selection_kind, selection_values = _resolve_blob_selection(
             ids, addresses, indices
@@ -2263,6 +2281,74 @@ def read_blobs(
             )
         return self._ds.read_blobs_by_indices(selection_values, blob_column, **kwargs)
 
+    def read_blob_ranges(
+        self,
+        blob_column: str,
+        requests: Sequence[Tuple[int, int, int]],
+        *,
+        selector: Literal["ids", "addresses", "indices"],
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, int, Optional[bytes]]]:
+        """
+        Read row-specific blob-local byte ranges with one planned API call.
+
+        Each request is a ``(row, offset, length)`` tuple. ``selector`` defines
+        whether every ``row`` value is interpreted as a stable row ID, physical
+        row address, or dataset index. Repeat a row value to read multiple ranges
+        from the same blob. Planning, range validation, physical-source grouping,
+        request coalescing, and bounded I/O scheduling all happen in Rust; this
+        method does not use a Python thread pool.
+
+        Every request produces one result. Requests on null blobs return ``None``,
+        including when the requested range is empty. Empty ranges on non-null
+        blobs return empty bytes without issuing payload I/O. For every request,
+        offset plus length must fit in an unsigned 64-bit integer. Ranges on
+        non-null blobs must not extend beyond the logical blob size. Blob-local
+        bounds are not evaluated for null values because they have no logical
+        payload length. By default results preserve the input request order.
+
+        Parameters
+        ----------
+        blob_column : str
+            The name of the blob column to read.
+        requests : Sequence[Tuple[int, int, int]]
+            Complete ``(row, offset, length)`` requests.
+        selector : {"ids", "addresses", "indices"}
+            Interpretation of every request's ``row`` value.
+        io_buffer_size : int, optional
+            Override the scheduler I/O buffer size used while materializing ranges.
+        preserve_order : bool, optional
+            If True, returned results follow request order. If False,
+            ``request_index`` still identifies every result.
+
+        Examples
+        --------
+        Read two disjoint ranges from the same row index:
+
+        .. code-block:: python
+
+            dataset.read_blob_ranges(
+                "images",
+                requests=[(7, 0, 1024), (7, 4096, 1024)],
+                selector="indices",
+            )
+
+        Returns
+        -------
+        results : List[Tuple[int, int, Optional[bytes]]]
+            One ``(request_index, row_address, data)`` tuple per request.
+            The Python list retains all returned payload bytes in memory; peak
+            result memory is therefore proportional to their total logical size.
+        """
+        return self._ds.read_blob_ranges(
+            list(requests),
+            blob_column,
+            selector,
+            io_buffer_size=io_buffer_size,
+            preserve_order=preserve_order,
+        )
+
     def head(self, num_rows, **kwargs):
         """
         Load the first N rows of the dataset.
@@ -3221,7 +3307,9 @@ def _is_segment_native_scalar_index_type(
             "BITMAP",
             "INVERTED",
             "FTS",
+            "RTREE",
             "ZONEMAP",
+            "BLOOMFILTER",
         }
 
     @classmethod
@@ -3232,7 +3320,9 @@ def _requires_uncommitted_scalar_index(
         return cls._normalized_index_type(index_type) in {
             "BTREE",
             "BITMAP",
+            "RTREE",
             "ZONEMAP",
+            "BLOOMFILTER",
         }
 
     def create_scalar_index(
@@ -3366,8 +3456,10 @@ def create_scalar_index(
         format_version: int or str, optional
             This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS
             format version to write when creating a new index. Accepts ``1``,
-            ``2``, ``"v1"``, or ``"v2"``. If unset, Lance chooses the current
-            default format.
+            ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``.
+            If unset, Lance uses ``LANCE_FTS_FORMAT_VERSION`` when present and
+            otherwise writes v2 for text analysis with ``block_size=128`` and
+            v3 for code analysis or ``block_size=256``.
 
         with_position: bool, default False
             This is for the ``INVERTED`` index. If True, the index will store the
@@ -3375,6 +3467,12 @@ def create_scalar_index(
             query. This will significantly increase the index size.
             It won't impact the performance of non-phrase queries even if it is set to
             True.
+        block_size: int, default 128
+            This is for the ``INVERTED`` index. Number of documents per compressed
+            posting block. Must be one of ``128`` or ``256``.
+            ``block_size=256`` is experimental and may introduce breaking changes.
+            Use ``128`` when stable compatibility with the legacy posting layout is
+            required.
         memory_limit: int, optional
             This is for the ``INVERTED`` index. Total build-time memory limit in MiB.
             If set, Lance divides this budget evenly across the workers. If unset,
@@ -4180,8 +4278,8 @@ def create_index_uncommitted(
         Create one segment without publishing it and return its metadata.
 
         This is the public distributed-build API for vector, BTREE scalar,
-        canonical bitmap scalar, INVERTED scalar, and ZONEMAP scalar index
-        construction. Unlike
+        canonical bitmap scalar, INVERTED scalar, RTREE scalar, ZONEMAP scalar,
+        and BLOOMFILTER scalar index construction. Unlike
         :meth:`create_index`, this method does not publish the index into the
         dataset manifest. Instead, it writes one segment under
         ``_indices/<segment_uuid>/`` and returns the resulting
@@ -4197,7 +4295,7 @@ def create_index_uncommitted(
         4. commit the final segment list with
            :meth:`commit_existing_index_segments`
 
-        BTREE, BITMAP, INVERTED, and ZONEMAP segments may
+        BTREE, BITMAP, INVERTED, RTREE, ZONEMAP, and BLOOMFILTER segments may
         be merged with :meth:`merge_existing_index_segments` before commit.
         Parameters are the same as :meth:`create_index`, with one additional
         requirement:
@@ -4294,13 +4392,22 @@ def drop_index(self, name: str):
         """
         return self._ds.drop_index(name)
 
-    def prewarm_index(self, name: str, *, with_position: bool = False):
+    def prewarm_index(
+        self,
+        name: str,
+        *,
+        with_position: bool = False,
+        index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
+    ):
         """
         Prewarm an index
 
-        This will load the entire index into memory.  This can help avoid cold start
-        issues with index queries.  If the index does not fit in the index cache, then
-        this will result in wasted I/O.
+        By default, this will load the entire index into memory. This can help
+        avoid cold start issues with index queries. If the index does not fit in
+        the index cache, then this will result in wasted I/O.
+
+        Use ``session().index_cache_size_bytes()`` before and after prewarm to
+        inspect how much the index cache grew.
 
         Parameters
         ----------
@@ -4310,8 +4417,16 @@ def prewarm_index(self, name: str, *, with_position: bool = False):
             This is only supported for ``INVERTED`` indices. If True, positions are
             also loaded into the cache during prewarm so phrase queries do not need a
             separate lazy positions read.
-        """
-        return self._ds.prewarm_index(name, with_position=with_position)
+        index_segments: iterable of str or uuid.UUID, default None
+            If specified, prewarm only these physical index segment UUIDs from the
+            named logical index. Use :meth:`describe_indices` to inspect logical
+            indices and obtain segment UUIDs from ``IndexDescription.segments``.
+        """
+        return self._ds.prewarm_index(
+            name,
+            with_position=with_position,
+            index_segments=_normalize_index_segment_ids(index_segments),
+        )
 
     def merge_index_metadata(
         self,
@@ -5057,7 +5172,6 @@ def initialize_mem_wal(
         identity_column: Optional[str] = None,
         unsharded: bool = False,
         durable_write: Optional[bool] = None,
-        sync_indexed_write: Optional[bool] = None,
         max_wal_buffer_size: Optional[int] = None,
         max_wal_flush_interval_ms: Optional[int] = None,
         max_memtable_size: Optional[int] = None,
@@ -5065,8 +5179,6 @@ def initialize_mem_wal(
         max_memtable_batches: Optional[int] = None,
         max_unflushed_memtable_bytes: Optional[int] = None,
         manifest_scan_batch_size: Optional[int] = None,
-        async_index_buffer_rows: Optional[int] = None,
-        async_index_interval_ms: Optional[int] = None,
         backpressure_log_interval_ms: Optional[int] = None,
         stats_log_interval_ms: Optional[int] = None,
         hnsw_params: Optional[Dict[str, Dict[str, int]]] = None,
@@ -5127,7 +5239,6 @@ def initialize_mem_wal(
             identity_column=identity_column,
             unsharded=unsharded,
             durable_write=durable_write,
-            sync_indexed_write=sync_indexed_write,
             max_wal_buffer_size=max_wal_buffer_size,
             max_wal_flush_interval_ms=max_wal_flush_interval_ms,
             max_memtable_size=max_memtable_size,
@@ -5135,8 +5246,6 @@ def initialize_mem_wal(
             max_memtable_batches=max_memtable_batches,
             max_unflushed_memtable_bytes=max_unflushed_memtable_bytes,
             manifest_scan_batch_size=manifest_scan_batch_size,
-            async_index_buffer_rows=async_index_buffer_rows,
-            async_index_interval_ms=async_index_interval_ms,
             backpressure_log_interval_ms=backpressure_log_interval_ms,
             stats_log_interval_ms=stats_log_interval_ms,
             hnsw_params=hnsw_params,
@@ -5159,7 +5268,6 @@ def mem_wal_writer(
         shard_id: str,
         *,
         durable_write: Optional[bool] = None,
-        sync_indexed_write: Optional[bool] = None,
         max_wal_buffer_size: Optional[int] = None,
         max_wal_flush_interval_ms: Optional[int] = None,
         max_memtable_size: Optional[int] = None,
@@ -5167,8 +5275,6 @@ def mem_wal_writer(
         max_memtable_batches: Optional[int] = None,
         max_unflushed_memtable_bytes: Optional[int] = None,
         manifest_scan_batch_size: Optional[int] = None,
-        async_index_buffer_rows: Optional[int] = None,
-        async_index_interval_ms: Optional[int] = None,
         backpressure_log_interval_ms: Optional[int] = None,
         stats_log_interval_ms: Optional[int] = None,
         hnsw_params: Optional[Dict[str, Dict[str, int]]] = None,
@@ -5186,8 +5292,6 @@ def mem_wal_writer(
             ``str(uuid.uuid4())``).
         durable_write : bool, optional
             Whether to fsync WAL writes (default: ``True``).
-        sync_indexed_write : bool, optional
-            Whether index updates are synchronous (default: ``True``).
         max_wal_buffer_size : int, optional
             Maximum WAL buffer size in bytes (default: 10 MB).
         max_wal_flush_interval_ms : int, optional
@@ -5202,10 +5306,6 @@ def mem_wal_writer(
             Maximum unflushed bytes before backpressure (default: 1 GB).
         manifest_scan_batch_size : int, optional
             Batch size for manifest scans (default: 2).
-        async_index_buffer_rows : int, optional
-            Buffer rows for async index updates (default: 10 000).
-        async_index_interval_ms : int, optional
-            Interval for async index updates in milliseconds (default: 1000).
         backpressure_log_interval_ms : int, optional
             Interval for backpressure log messages in milliseconds
             (default: 30 000).
@@ -5253,7 +5353,6 @@ def mem_wal_writer(
             name: val
             for name, val in [
                 ("durable_write", durable_write),
-                ("sync_indexed_write", sync_indexed_write),
                 ("max_wal_buffer_size", max_wal_buffer_size),
                 ("max_wal_flush_interval_ms", max_wal_flush_interval_ms),
                 ("max_memtable_size", max_memtable_size),
@@ -5261,8 +5360,6 @@ def mem_wal_writer(
                 ("max_memtable_batches", max_memtable_batches),
                 ("max_unflushed_memtable_bytes", max_unflushed_memtable_bytes),
                 ("manifest_scan_batch_size", manifest_scan_batch_size),
-                ("async_index_buffer_rows", async_index_buffer_rows),
-                ("async_index_interval_ms", async_index_interval_ms),
                 ("backpressure_log_interval_ms", backpressure_log_interval_ms),
                 ("stats_log_interval_ms", stats_log_interval_ms),
                 ("hnsw_params", hnsw_params),
@@ -5962,6 +6059,76 @@ class DataReplacement(BaseOperation):
 
         replacements: List[LanceOperation.DataReplacementGroup]
 
+    @dataclass
+    class DataOverlayFile:
+        """
+        An overlay file supplying new values for a subset of
+        ``(physical offset, field)`` cells of a fragment, resolved on read and
+        layered over the base data without rewriting the base files.
+
+        The overlay is dense or sparse depending on the shape of ``offsets``:
+        pass a flat ``List[int]`` for a dense overlay (one offset list shared by
+        every field in ``data_file``) or a ``List[List[int]]`` for a sparse
+        overlay (one offset list per field, in the order of the file's fields).
+        Offsets are **physical** row offsets (positions in the base files,
+        counting deleted rows), like deletion vectors.
+
+        Attributes
+        ----------
+        data_file : DataFile
+            The Lance data file storing the overlay's new cell values — one
+            value column per covered field. The value at each covered offset is
+            stored at the rank (0-based count of covered offsets below it) of
+            that offset in the field's coverage.
+        offsets : Union[List[int], List[List[int]]]
+            The covered physical row offsets. A flat list is dense coverage
+            (shared by every field); a list of per-field lists is sparse
+            coverage (in field order). Each list must be strictly ascending
+            with no duplicates, since the Nth offset maps to the Nth value row
+            in ``data_file``; a non-ascending list raises ``ValueError``.
+        committed_version : Optional[int]
+            The dataset version at which this overlay became effective. Leave as
+            ``None`` when creating an overlay to commit — the commit stamps it.
+            It is populated when reading an existing fragment's overlays so they
+            round-trip through :class:`FragmentMetadata`.
+        """
+
+        data_file: DataFile
+        offsets: Union[List[int], List[List[int]]]
+        committed_version: Optional[int] = None
+
+    @dataclass
+    class DataOverlayGroup:
+        """
+        Overlay files to append to a single fragment.
+
+        Attributes
+        ----------
+        fragment_id : int
+            The id of the fragment the overlays apply to.
+        overlays : List[LanceOperation.DataOverlayFile]
+            The overlay files to append, ordered oldest-first (a later entry is
+            newer and wins where coverage overlaps).
+        """
+
+        fragment_id: int
+        overlays: List[LanceOperation.DataOverlayFile]
+
+    @dataclass
+    class DataOverlay(BaseOperation):
+        """
+        Operation that appends data overlay files to fragments.
+
+        Overlays are appended to each fragment's existing overlays (overlays
+        written by concurrent commits are preserved) and resolved on read
+        over the base data without rewriting it.
+
+        If multiple groups target the same data then the values in the
+        latest group take precedence.
+        """
+
+        groups: List[LanceOperation.DataOverlayGroup]
+
     @dataclass
     class Project(BaseOperation):
         """
@@ -6182,10 +6349,12 @@ def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder:
 
     def batch_readahead(self, nbatches: Optional[int] = None) -> ScannerBuilder:
         """
-        This parameter is ignored when reading v2 files
+        Set the maximum number of batches to decode concurrently.
+
+        This parameter must be greater than zero.
         """
-        if nbatches is not None and int(nbatches) < 0:
-            raise ValueError("batch_readahead must be non-negative")
+        if nbatches is not None and int(nbatches) <= 0:
+            raise ValueError("batch_readahead must be greater than 0")
         self._batch_readahead = nbatches
         return self
 
@@ -6394,19 +6563,7 @@ def with_fragments(
     def with_index_segments(
         self, index_segments: Optional[Iterable[Union[str, uuid.UUID]]]
     ) -> ScannerBuilder:
-        if index_segments is not None:
-            segment_ids = []
-            for segment_id in index_segments:
-                if isinstance(segment_id, (str, uuid.UUID)):
-                    segment_ids.append(str(segment_id))
-                else:
-                    raise TypeError(
-                        "index_segments must be an iterable of str or uuid.UUID. "
-                        f"Got {type(segment_id)} instead."
-                    )
-            index_segments = segment_ids
-
-        self._index_segments = index_segments
+        self._index_segments = _normalize_index_segment_ids(index_segments)
         return self
 
     def nearest(
@@ -7564,8 +7721,7 @@ def _coerce_query_vector(query: QueryVectorLike) -> tuple[pa.Array, int]:
         if isinstance(query.type, pa.FixedSizeListType):
             query = query.values
     elif isinstance(query, (list, tuple)) or (
-        _check_for_numpy(query),
-        isinstance(query, np.ndarray),
+        _check_for_numpy(query) and isinstance(query, np.ndarray)
     ):
         query = np.array(query).astype("float64")  # workaround for GH-608
         query = pa.FloatingPointArray.from_pandas(query, type=pa.float32())
diff --git a/python/python/lance/dependencies.py b/python/python/lance/dependencies.py
index def9a052504..a739b7bc75d 100644
--- a/python/python/lance/dependencies.py
+++ b/python/python/lance/dependencies.py
@@ -27,7 +27,6 @@
 _CAGRA_AVAILABLE = True
 _RAFT_COMMON_AVAILABLE = True
 _HUGGING_FACE_AVAILABLE = True
-_TENSORFLOW_AVAILABLE = True
 
 
 class _LazyModule(ModuleType):
@@ -49,7 +48,6 @@ class _LazyModule(ModuleType):
         "pandas": "pd.",
         "polars": "pl.",
         "torch": "torch.",
-        "tensorflow": "tf.",
     }
 
     def __init__(
@@ -163,7 +161,6 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]:
     import numpy
     import pandas
     import polars
-    import tensorflow  # type: ignore[reportMissingImports]
     import torch  # type: ignore[reportMissingImports]
 else:
     # heavy/optional third party libs
@@ -172,7 +169,6 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]:
     polars, _POLARS_AVAILABLE = _lazy_import("polars")
     torch, _TORCH_AVAILABLE = _lazy_import("torch")
     datasets, _HUGGING_FACE_AVAILABLE = _lazy_import("datasets")
-    tensorflow, _TENSORFLOW_AVAILABLE = _lazy_import("tensorflow")
 
 
 @lru_cache(maxsize=None)
@@ -215,26 +211,18 @@ def _check_for_hugging_face(obj: Any, *, check_type: bool = True) -> bool:
     )
 
 
-def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool:
-    return _TENSORFLOW_AVAILABLE and _might_be(
-        cast("Hashable", type(obj) if check_type else obj), "tensorflow"
-    )
-
-
 __all__ = [
     # lazy-load third party libs
     "datasets",
     "numpy",
     "pandas",
     "polars",
-    "tensorflow",
     "torch",
     # lazy utilities
     "_check_for_hugging_face",
     "_check_for_numpy",
     "_check_for_pandas",
     "_check_for_polars",
-    "_check_for_tensorflow",
     "_check_for_torch",
     "_LazyModule",
     # exported flags/guards
@@ -243,5 +231,4 @@ def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool:
     "_POLARS_AVAILABLE",
     "_TORCH_AVAILABLE",
     "_HUGGING_FACE_AVAILABLE",
-    "_TENSORFLOW_AVAILABLE",
 ]
diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py
index adf220e59f6..bca62e2b280 100644
--- a/python/python/lance/fragment.py
+++ b/python/python/lance/fragment.py
@@ -45,6 +45,7 @@
         ColumnOrdering,
         DatasetBasePath,
         LanceDataset,
+        LanceOperation,
         LanceScanner,
         ReaderLike,
         Transaction,
@@ -78,6 +79,11 @@ class FragmentMetadata:
         The row created at version metadata, if any.
     last_updated_at_version_meta : Optional[RowDatasetVersionMeta]
         The row last updated at version metadata, if any.
+    overlays : List[LanceOperation.DataOverlayFile]
+        The data overlay files layered over this fragment's base data, if any.
+        Overlays are created via :class:`LanceOperation.DataOverlay`; they are
+        carried here so they survive operations that round-trip fragment
+        metadata (e.g. a manual ``Delete``, ``Update``, or ``Merge`` commit).
     """
 
     id: int
@@ -87,6 +93,7 @@ class FragmentMetadata:
     row_id_meta: Optional[RowIdMeta] = None
     created_at_version_meta: Optional[RowDatasetVersionMeta] = None
     last_updated_at_version_meta: Optional[RowDatasetVersionMeta] = None
+    overlays: List["LanceOperation.DataOverlayFile"] = field(default_factory=list)
 
     @property
     def num_deletions(self) -> int:
@@ -110,12 +117,25 @@ def data_files(self) -> List[DataFile]:
 
     def to_json(self) -> dict:
         """Get this as a simple JSON-serializable dictionary."""
-        files = [asdict(f) for f in self.files]
-        for f in files:
-            f["path"] = f.pop("_path")
+
+        def _data_file_to_json(f: DataFile) -> dict:
+            d = asdict(f)
+            d["path"] = d.pop("_path")
+            return d
+
+        files = [_data_file_to_json(f) for f in self.files]
+        overlays = [
+            dict(
+                data_file=_data_file_to_json(o.data_file),
+                offsets=o.offsets,
+                committed_version=o.committed_version,
+            )
+            for o in self.overlays
+        ]
         return dict(
             id=self.id,
             files=files,
+            overlays=overlays,
             physical_rows=self.physical_rows,
             deletion_file=(
                 self.deletion_file.asdict() if self.deletion_file is not None else None
@@ -159,6 +179,20 @@ def from_json(json_data: str) -> FragmentMetadata:
                 json.dumps(last_updated_at_version_meta)
             )
 
+        overlays = []
+        overlays_json = json_data.get("overlays")
+        if overlays_json:
+            from .dataset import LanceOperation
+
+            overlays = [
+                LanceOperation.DataOverlayFile(
+                    data_file=DataFile(**o["data_file"]),
+                    offsets=o["offsets"],
+                    committed_version=o.get("committed_version"),
+                )
+                for o in overlays_json
+            ]
+
         return FragmentMetadata(
             id=json_data["id"],
             files=[DataFile(**f) for f in json_data["files"]],
@@ -167,6 +201,7 @@ def from_json(json_data: str) -> FragmentMetadata:
             row_id_meta=row_id_meta,
             created_at_version_meta=created_at_version_meta,
             last_updated_at_version_meta=last_updated_at_version_meta,
+            overlays=overlays,
         )
 
 
diff --git a/python/python/lance/indices/builder.py b/python/python/lance/indices/builder.py
index 6059166d6ba..e235f348979 100644
--- a/python/python/lance/indices/builder.py
+++ b/python/python/lance/indices/builder.py
@@ -164,6 +164,7 @@ def train_pq(
         *,
         sample_rate: int = 256,
         max_iters: int = 50,
+        num_bits: int = 8,
         fragment_ids: Optional[list[int]] = None,
     ) -> PqModel:
         """
@@ -195,6 +196,8 @@ def train_pq(
             This parameter is used in the same way as in the IVF model.
         max_iters: int
             This parameter is used in the same way as in the IVF model.
+        num_bits: int
+            The number of bits used to encode each PQ centroid.
         fragment_ids: list[int], optional
             If provided, train using only the specified fragments from the dataset.
         """
@@ -202,7 +205,7 @@ def train_pq(
 
         num_rows = self._count_rows(fragment_ids)
         num_subvectors = self._normalize_pq_params(num_subvectors, self.dimension)
-        self._verify_pq_sample_rate(num_rows, sample_rate)
+        self._verify_pq_sample_rate(num_rows, sample_rate, num_bits)
         distance_type = ivf_model.distance_type
         pq_codebook = indices.train_pq_model(
             self.dataset._ds,
@@ -214,8 +217,9 @@ def train_pq(
             max_iters,
             ivf_model.centroids,
             fragment_ids,
+            num_bits,
         )
-        return PqModel(num_subvectors, pq_codebook)
+        return PqModel(num_subvectors, pq_codebook, num_bits=num_bits)
 
     def prepare_global_ivf_pq(
         self,
@@ -226,6 +230,7 @@ def prepare_global_ivf_pq(
         accelerator: Optional[Union[str, "torch.Device"]] = None,
         sample_rate: int = 256,
         max_iters: int = 50,
+        num_bits: int = 8,
         fragment_ids: Optional[list[int]] = None,
     ) -> dict:
         """
@@ -267,6 +272,7 @@ def prepare_global_ivf_pq(
             num_subvectors,
             sample_rate=sample_rate,
             max_iters=max_iters,
+            num_bits=num_bits,
             fragment_ids=fragment_ids,
         )
 
@@ -381,6 +387,7 @@ def transform_vectors(
             dest_uri,
             fragments,
             partition_ds_uri,
+            pq.num_bits,
         )
 
     def shuffle_transformed_vectors(
@@ -471,6 +478,7 @@ def load_shuffled_vectors(
                 num_subvectors,
                 distance_type,
                 index_name,
+                pq.num_bits,
             )
         else:
             raise ValueError("filenames must be a list of strings")
@@ -526,13 +534,17 @@ def _verify_base_sample_rate(self, sample_rate: int):
                 f"The sample_rate must be an int greater than 1, got {sample_rate}"
             )
 
-    def _verify_pq_sample_rate(self, num_rows: int, sample_rate: int):
+    def _verify_pq_sample_rate(
+        self, num_rows: int, sample_rate: int, num_bits: int = 8
+    ):
         self._verify_base_sample_rate(sample_rate)
-        if 256 * sample_rate > num_rows:
+        required_rows = (2**num_bits) * sample_rate
+        if required_rows > num_rows:
             raise ValueError(
                 "There are not enough rows in the dataset to create PQ"
-                f" codebook with a sample rate of {sample_rate}.  {sample_rate * 256}"
-                f" rows needed and there are {num_rows}"
+                f" codebook with a sample rate of {sample_rate} and num_bits"
+                f" of {num_bits}.  {required_rows} rows needed and there are"
+                f" {num_rows}"
             )
 
     def _verify_ivf_sample_rate(
diff --git a/python/python/lance/indices/pq.py b/python/python/lance/indices/pq.py
index b3aeb50bcbe..e3d334ccf48 100644
--- a/python/python/lance/indices/pq.py
+++ b/python/python/lance/indices/pq.py
@@ -14,9 +14,13 @@ class PqModel:
     Can be saved / loaded to checkpoint progress.
     """
 
-    def __init__(self, num_subvectors: int, codebook: pa.FixedSizeListArray):
+    def __init__(
+        self, num_subvectors: int, codebook: pa.FixedSizeListArray, *, num_bits: int = 8
+    ):
         self.num_subvectors = num_subvectors
         """The number of subvectors to divide source vectors into"""
+        self.num_bits = num_bits
+        """The number of bits used to encode each PQ centroid"""
         self.codebook = codebook
         """The centroids of the PQ clusters"""
 
@@ -42,7 +46,10 @@ def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
             uri,
             pa.schema(
                 [pa.field("codebook", self.codebook.type)],
-                metadata={b"num_subvectors": str(self.num_subvectors).encode()},
+                metadata={
+                    b"num_subvectors": str(self.num_subvectors).encode(),
+                    b"num_bits": str(self.num_bits).encode(),
+                },
             ),
             storage_options=storage_options,
         ) as writer:
@@ -65,9 +72,10 @@ def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         reader = LanceFileReader(uri, storage_options=storage_options)
         num_rows = reader.metadata().num_rows
-        metadata = reader.metadata().schema.metadata
+        metadata = reader.metadata().schema.metadata or {}
         num_subvectors = int(metadata[b"num_subvectors"].decode())
+        num_bits = int(metadata.get(b"num_bits", b"8").decode())
         codebook = (
             reader.read_all(batch_size=num_rows).to_table().column("codebook").chunk(0)
         )
-        return cls(num_subvectors, codebook)
+        return cls(num_subvectors, codebook, num_bits=num_bits)
diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi
index f050c4c8422..9ba76cc592d 100644
--- a/python/python/lance/lance/__init__.pyi
+++ b/python/python/lance/lance/__init__.pyi
@@ -87,6 +87,25 @@ from .trace import capture_trace_events as capture_trace_events
 from .trace import shutdown_tracing as shutdown_tracing
 from .trace import trace_to_chrome as trace_to_chrome
 
+class MetricPoint:
+    name: str
+    kind: str
+    attributes: Dict[str, str]
+    value: Optional[float]
+    buckets: Optional[List[Tuple[str, int]]]
+    count: Optional[int]
+    sum: Optional[float]
+
+class MetricDescription:
+    name: str
+    kind: str
+    unit: Optional[str]
+    description: str
+
+def register_lance_metrics_recorder() -> bool: ...
+def lance_metrics_catalog() -> List[MetricDescription]: ...
+def snapshot_lance_metrics() -> List[MetricPoint]: ...
+
 class CleanupStats:
     bytes_removed: int
     old_versions: int
@@ -141,8 +160,15 @@ class PackedBlobWriter:
     def blob_id(self) -> int: ...
     @property
     def path(self) -> str: ...
+    @property
+    def field(self) -> pa.Field: ...
     def write_blob(self, data: bytes) -> None: ...
+    def write_blobs(
+        self,
+        payloads: Union[pa.BinaryArray, pa.LargeBinaryArray, pa.ChunkedArray],
+    ) -> None: ...
     def finish(self) -> List[BlobDescriptor]: ...
+    def finish_array(self, field_name: str) -> pa.StructArray: ...
 
 class DedicatedBlobWriter:
     @property
@@ -253,6 +279,7 @@ class LanceColumnStatistics:
 
 class _Session:
     def size_bytes(self) -> int: ...
+    def index_cache_size_bytes(self) -> int: ...
 
 class LanceBlobFile:
     def close(self): ...
@@ -344,38 +371,46 @@ class _Dataset:
         self,
         row_ids: List[int],
         blob_column: str,
-    ) -> List[LanceBlobFile]: ...
+    ) -> List[Optional[LanceBlobFile]]: ...
     def take_blobs_by_addresses(
         self,
         row_addresses: List[int],
         blob_column: str,
-    ) -> List[LanceBlobFile]: ...
+    ) -> List[Optional[LanceBlobFile]]: ...
     def take_blobs_by_indices(
         self,
         row_indices: List[int],
         blob_column: str,
-    ) -> List[LanceBlobFile]: ...
+    ) -> List[Optional[LanceBlobFile]]: ...
     def read_blobs(
         self,
         row_ids: List[int],
         blob_column: str,
         io_buffer_size: Optional[int] = None,
         preserve_order: Optional[bool] = None,
-    ) -> List[Tuple[int, bytes]]: ...
+    ) -> List[Tuple[int, Optional[bytes]]]: ...
     def read_blobs_by_addresses(
         self,
         row_addresses: List[int],
         blob_column: str,
         io_buffer_size: Optional[int] = None,
         preserve_order: Optional[bool] = None,
-    ) -> List[Tuple[int, bytes]]: ...
+    ) -> List[Tuple[int, Optional[bytes]]]: ...
     def read_blobs_by_indices(
         self,
         row_indices: List[int],
         blob_column: str,
         io_buffer_size: Optional[int] = None,
         preserve_order: Optional[bool] = None,
-    ) -> List[Tuple[int, bytes]]: ...
+    ) -> List[Tuple[int, Optional[bytes]]]: ...
+    def read_blob_ranges(
+        self,
+        requests: List[Tuple[int, int, int]],
+        blob_column: str,
+        selector: Literal["ids", "addresses", "indices"],
+        io_buffer_size: Optional[int] = None,
+        preserve_order: Optional[bool] = None,
+    ) -> List[Tuple[int, int, Optional[bytes]]]: ...
     def take_scan(
         self,
         row_slices: Iterable[Tuple[int, int]],
@@ -470,7 +505,13 @@ class _Dataset:
         kwargs: Optional[Dict[str, Any]] = None,
     ): ...
     def drop_index(self, name: str): ...
-    def prewarm_index(self, name: str, *, with_position: bool = False): ...
+    def prewarm_index(
+        self,
+        name: str,
+        *,
+        with_position: bool = False,
+        index_segments: Optional[List[str]] = None,
+    ): ...
     def merge_index_metadata(
         self,
         index_uuid: str,
@@ -542,8 +583,13 @@ class _Dataset:
         index_name: str,
         partition_id: int,
         hamming_threshold: int,
+        index_segments: Optional[List[str]] = None,
     ) -> pa.RecordBatchReader: ...
-    def get_ivf_partition_info(self, index_name: str) -> List[dict]: ...
+    def get_ivf_partition_info(
+        self,
+        index_name: str,
+        index_segments: Optional[List[str]] = None,
+    ) -> List[dict]: ...
     def hamming_clustering_for_sample(
         self,
         column: str,
@@ -568,6 +614,13 @@ class _MergeInsertBuilder:
     def target_bases(self, bases: list[str]) -> Self: ...
     def target_all_bases(self, include_primary: bool = True) -> Self: ...
     def execute(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ...
+    def execute_batches(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ...
+    def execute_uncommitted(
+        self, new_data: pa.RecordBatchReader
+    ) -> tuple[Transaction, ExecuteResult]: ...
+    def execute_uncommitted_batches(
+        self, new_data: pa.RecordBatchReader
+    ) -> tuple[Transaction, ExecuteResult]: ...
 
 class _Scanner:
     @property
@@ -690,7 +743,7 @@ def _evaluate_sharding_spec(
     schema: LanceSchema,
 ) -> pa.RecordBatch: ...
 
-class _MergedGeneration:
+class _CompactedSsTable:
     shard_id: str
     generation: int
     def __init__(self, shard_id: str, generation: int) -> None: ...
@@ -700,11 +753,12 @@ class _ShardSnapshot:
     def __init__(self, shard_id: str) -> None: ...
     def with_spec_id(self, spec_id: int) -> Self: ...
     def with_current_generation(self, generation: int) -> Self: ...
-    def with_flushed_generation(self, generation: int, path: str) -> Self: ...
+    def with_sstable(self, generation: int, path: str) -> Self: ...
 
 class _ShardWriter:
     shard_id: str
     def put(self, data: Any) -> None: ...
+    def delete(self, keys: Any) -> None: ...
     def close(self) -> None: ...
     def stats(self) -> Dict[str, Any]: ...
     def memtable_stats(self) -> Dict[str, Any]: ...
diff --git a/python/python/lance/lance/indices/__init__.pyi b/python/python/lance/lance/indices/__init__.pyi
index 0f5db7037df..181ec93b2d1 100644
--- a/python/python/lance/lance/indices/__init__.pyi
+++ b/python/python/lance/lance/indices/__init__.pyi
@@ -17,6 +17,8 @@ from typing import Optional
 
 import pyarrow as pa
 
+from .. import _Fragment
+
 class IndexConfig:
     index_type: str
     config: str
@@ -48,6 +50,7 @@ def train_pq_model(
     max_iters: int,
     ivf_model: pa.Array,
     fragment_ids: Optional[list[int]] = None,
+    num_bits: int = 8,
 ) -> pa.Array: ...
 def transform_vectors(
     dataset,
@@ -58,6 +61,9 @@ def transform_vectors(
     ivf_centroids: pa.Array,
     pq_codebook: pa.Array,
     dst_uri: str,
+    fragments: list[_Fragment],
+    partitions_ds_uri: Optional[str] = None,
+    num_bits: int = 8,
 ): ...
 def build_rq_model(
     dimension: int,
diff --git a/python/python/lance/mem_wal.py b/python/python/lance/mem_wal.py
index f87e811f830..325d97607e3 100644
--- a/python/python/lance/mem_wal.py
+++ b/python/python/lance/mem_wal.py
@@ -9,8 +9,8 @@
 
 1. **WAL** – append-only durable log (raw writes)
 2. **Active MemTable** – in-memory write buffer
-3. **Flushed MemTable** – Lance files written to object store
-4. **Base table** – canonical Lance dataset files (after merge_insert)
+3. **SSTable** – Lance files written to object store
+4. **Base table** – canonical Lance dataset files (after SSTable compaction)
 """
 
 from __future__ import annotations
@@ -22,12 +22,12 @@
 import pyarrow as pa
 
 from .lance import (
+    _CompactedSsTable,
     _evaluate_sharding_spec,
     _ExecutionPlan,
     _LsmPointLookupPlanner,
     _LsmScanner,
     _LsmVectorSearchPlanner,
-    _MergedGeneration,
     _ShardSnapshot,
     _ShardWriter,
 )
@@ -40,7 +40,7 @@
     "ShardingField",
     "ShardingSpec",
     "evaluate_sharding_spec",
-    "MergedGeneration",
+    "CompactedSsTable",
     "ShardSnapshot",
     "ShardWriter",
     "LsmScanner",
@@ -124,19 +124,19 @@ def _sharding_spec_to_dict(spec: Union[ShardingSpec, Mapping[str, object]]) -> d
 
 
 @dataclass
-class MergedGeneration:
-    """Identifies a flushed MemWAL generation that has been merged.
+class CompactedSsTable:
+    """Points to an SSTable compacted into the base table.
 
-    Pass a list of these to mark_generations_as_merged
-    so Lance knows which generations are now in the base table.
+    Pass a list of these to mark_sstables_as_compacted
+    so Lance can record compaction progress.
 
     Parameters
     ----------
     shard_id : str
         UUID string for the write shard.
     generation : int
-        Generation number (from
-        :attr:`ShardSnapshot.flushed_generations`).
+        Generation number of the compacted SSTable (as passed to
+        :meth:`ShardSnapshot.with_sstable`).
     """
 
     shard_id: str
@@ -170,9 +170,9 @@ def with_current_generation(self, generation: int) -> "ShardSnapshot":
         self._raw = self._raw.with_current_generation(generation)
         return self
 
-    def with_flushed_generation(self, generation: int, path: str) -> "ShardSnapshot":
-        """Add a flushed generation with its storage path."""
-        self._raw = self._raw.with_flushed_generation(generation, path)
+    def with_sstable(self, generation: int, path: str) -> "ShardSnapshot":
+        """Add an SSTable with its storage path."""
+        self._raw = self._raw.with_sstable(generation, path)
         return self
 
     def __repr__(self) -> str:
@@ -187,6 +187,7 @@ class ShardWriter:
 
         with dataset.mem_wal_writer(shard_id) as writer:
             writer.put(batch)
+            writer.delete(pa.table({"id": [1]}))
 
     Parameters
     ----------
@@ -224,6 +225,36 @@ def put(self, data, *, schema: Optional[pa.Schema] = None) -> None:
         reader = _coerce_reader(data, schema)
         self._raw.put(reader)
 
+    def delete(self, keys, *, schema: Optional[pa.Schema] = None) -> None:
+        """Delete rows by primary key from the MemWAL.
+
+        Parameters
+        ----------
+        keys : ReaderLike
+            Any Arrow-compatible data containing this shard's primary key
+            column(s). Non-primary-key columns, if present, are ignored by
+            the Rust core delete path.
+        schema : pa.Schema, optional
+            Schema hint, needed when *keys* is a generator.
+
+        Raises
+        ------
+        IOError
+            If delete validation fails, WAL flush fails, or the writer has
+            already been closed. Delete validation is centralized in Rust and
+            includes checks for primary-key metadata and tombstone-compatible
+            nullable non-key columns.
+
+        Examples
+        --------
+        ::
+
+            with dataset.mem_wal_writer(shard_id) as writer:
+                writer.delete(pa.table({"id": [42]}))
+        """
+        reader = _coerce_reader(keys, schema)
+        self._raw.delete(reader)
+
     def close(self) -> None:
         """Flush and close the writer.
 
@@ -261,7 +292,7 @@ def lsm_scanner(
     ) -> "LsmScanner":
         """Create an LSM scanner that includes the active MemTable.
 
-        This scanner covers the base table, the given flushed generations,
+        This scanner covers the base table, the given SSTables,
         and the current active MemTable — providing strong read-your-writes
         consistency.
 
@@ -290,10 +321,10 @@ class LsmScanner:
     """LSM-aware scanner covering all data levels.
 
     Deduplicates by primary key, always returning the newest version of
-    each row across base table, flushed MemTables, and the active MemTable.
+    each row across base table, SSTables, and the active MemTable.
 
     Obtain an instance from `ShardWriter.lsm_scanner` (includes
-    active MemTable) or `LsmScanner.from_snapshots` (flushed only).
+    active MemTable) or `LsmScanner.from_snapshots` (SSTables only).
 
     The builder methods (`project`, `filter`, `limit`)
     return ``self`` for chaining.
@@ -323,7 +354,7 @@ def from_snapshots(
         dataset : LanceDataset
             The base dataset to scan.
         shard_snapshots : list of ShardSnapshot
-            Shard snapshots specifying flushed generations to include.
+            Shard snapshots specifying SSTables to include.
         """
         raw = _LsmScanner.from_snapshots(dataset._ds, [s._raw for s in shard_snapshots])
         return LsmScanner(raw)
@@ -425,7 +456,7 @@ class LsmPointLookupPlanner:
     dataset : LanceDataset
         The base dataset.
     shard_snapshots : list of ShardSnapshot
-        Shard snapshots specifying flushed generations to include.
+        Shard snapshots specifying SSTables to include.
     pk_columns : list of str, optional
         Primary key column names.  Inferred from schema metadata if omitted.
 
@@ -484,7 +515,7 @@ class LsmVectorSearchPlanner:
     dataset : LanceDataset
         The base dataset.
     shard_snapshots : list of ShardSnapshot
-        Shard snapshots specifying flushed generations to include.
+        Shard snapshots specifying SSTables to include.
     vector_column : str
         Name of the ``FixedSizeList<float32>`` vector column.
     pk_columns : list of str, optional
@@ -596,8 +627,8 @@ def _unwrap_shard_id(shard_id: str) -> str:
     return shard_id
 
 
-def _to_raw_merged_generations(
-    generations: Iterable[MergedGeneration],
+def _to_raw_compacted_sstables(
+    sstables: Iterable[CompactedSsTable],
 ) -> list:
-    """Convert Python MergedGeneration list to PyO3 _MergedGeneration list."""
-    return [_MergedGeneration(g.shard_id, g.generation) for g in generations]
+    """Convert Python CompactedSsTable list to PyO3 _CompactedSsTable list."""
+    return [_CompactedSsTable(s.shard_id, s.generation) for s in sstables]
diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py
index fec3a1cfb1e..6df5b9aa4bc 100644
--- a/python/python/lance/namespace.py
+++ b/python/python/lance/namespace.py
@@ -342,12 +342,13 @@ class DirectoryNamespace(LanceNamespace):
     >>>
     >>> # With AWS credential vending (requires credential-vendor-aws feature)
     >>> # Use **dict to pass property names with dots
-    >>> ns = lance.namespace.DirectoryNamespace(**{
+    >>> aws_properties = {
     ...     "root": "s3://my-bucket/data",
     ...     "credential_vendor.enabled": "true",
     ...     "credential_vendor.aws_role_arn": "arn:aws:iam::123456789012:role/MyRole",
     ...     "credential_vendor.aws_duration_millis": "3600000",
-    ... })
+    ... }
+    >>> # ns = lance.namespace.DirectoryNamespace(**aws_properties)
 
     With dynamic context provider:
 
diff --git a/python/python/lance/otel.py b/python/python/lance/otel.py
new file mode 100644
index 00000000000..0c9ee1790fd
--- /dev/null
+++ b/python/python/lance/otel.py
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Bridge Lance's internal metrics into OpenTelemetry.
+
+Lance core publishes metrics (currently object store request counts, bytes,
+latency, errors, and throttles) through the Rust ``metrics`` facade. This module
+installs a process-global recorder that aggregates them and registers
+OpenTelemetry observable instruments that report the aggregated values into the
+user's ``MeterProvider``.
+
+The bridge is generic: every metric Lance describes is surfaced automatically,
+with no per-metric Python code. Histograms have no asynchronous OpenTelemetry
+instrument, so each is exported Prometheus-style as cumulative ``le`` buckets
+plus ``_count`` and ``_sum`` observable counters.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import TYPE_CHECKING, Optional
+
+from .lance import (
+    lance_metrics_catalog,
+    register_lance_metrics_recorder,
+    snapshot_lance_metrics,
+)
+
+if TYPE_CHECKING:
+    from opentelemetry.metrics import MeterProvider
+
+_INSTRUMENTED = False
+
+
+def instrument_lance_metrics(meter_provider: Optional["MeterProvider"] = None) -> bool:
+    """Register Lance metrics as OpenTelemetry observable instruments.
+
+    Installs a process-global metrics recorder and creates one observable
+    instrument per Lance metric on the given (or global) ``MeterProvider``. The
+    user's configured ``MetricReader`` then collects them on its own schedule.
+
+    Counters and gauges map directly to observable counters/gauges. Each
+    histogram is exported as cumulative ``le`` bucket counts (``<name>_bucket``,
+    with an ``le`` attribute) plus ``<name>_count`` and ``<name>_sum``.
+
+    Parameters
+    ----------
+    meter_provider : opentelemetry.metrics.MeterProvider, optional
+        The provider to register instruments on. Defaults to the global provider
+        from ``opentelemetry.metrics.get_meter_provider()``.
+
+    Returns
+    -------
+    bool
+        ``True`` if the recorder is installed and instruments are registered.
+        ``False`` if a different ``metrics`` recorder is already installed in
+        this process (``metrics`` permits only one global recorder), in which
+        case a warning is emitted and no instruments are created.
+
+    Notes
+    -----
+    Requires the OpenTelemetry SDK (``pip install pylance[otel]``). Calling this
+    more than once is safe; instruments are created only on the first successful
+    call.
+    """
+    global _INSTRUMENTED
+
+    try:
+        from opentelemetry.metrics import Observation, get_meter_provider
+    except ImportError as exc:
+        raise ImportError(
+            "instrument_lance_metrics requires the OpenTelemetry API/SDK. "
+            "Install it with `pip install pylance[otel]` or "
+            "`pip install opentelemetry-sdk`."
+        ) from exc
+
+    if not register_lance_metrics_recorder():
+        warnings.warn(
+            "Could not install the Lance metrics recorder: another `metrics` "
+            "recorder is already installed in this process. Lance metrics will "
+            "not be exported via OpenTelemetry.",
+            stacklevel=2,
+        )
+        return False
+
+    if _INSTRUMENTED:
+        return True
+
+    provider = meter_provider or get_meter_provider()
+    meter = provider.get_meter("lance")
+
+    def scalar_callback(metric_name: str):
+        def callback(_options):
+            return [
+                Observation(point.value, point.attributes)
+                for point in snapshot_lance_metrics()
+                if point.name == metric_name and point.value is not None
+            ]
+
+        return callback
+
+    def bucket_callback(metric_name: str):
+        def callback(_options):
+            observations = []
+            for point in snapshot_lance_metrics():
+                if point.name != metric_name or point.buckets is None:
+                    continue
+                for le, cumulative in point.buckets:
+                    attributes = dict(point.attributes)
+                    attributes["le"] = le
+                    observations.append(Observation(cumulative, attributes))
+            return observations
+
+        return callback
+
+    def field_callback(metric_name: str, field: str):
+        def callback(_options):
+            observations = []
+            for point in snapshot_lance_metrics():
+                if point.name != metric_name:
+                    continue
+                value = getattr(point, field)
+                if value is not None:
+                    observations.append(Observation(value, point.attributes))
+            return observations
+
+        return callback
+
+    for desc in lance_metrics_catalog():
+        unit = desc.unit or ""
+        if desc.kind == "counter":
+            meter.create_observable_counter(
+                desc.name,
+                callbacks=[scalar_callback(desc.name)],
+                unit=unit,
+                description=desc.description,
+            )
+        elif desc.kind == "gauge":
+            meter.create_observable_gauge(
+                desc.name,
+                callbacks=[scalar_callback(desc.name)],
+                unit=unit,
+                description=desc.description,
+            )
+        elif desc.kind == "histogram":
+            # `_bucket` and `_count` observe cumulative counts, so they are
+            # unitless; only `_sum` carries the histogram's unit (e.g. seconds).
+            meter.create_observable_counter(
+                f"{desc.name}_bucket",
+                callbacks=[bucket_callback(desc.name)],
+                description=f"{desc.description} (cumulative buckets)",
+            )
+            meter.create_observable_counter(
+                f"{desc.name}_count",
+                callbacks=[field_callback(desc.name, "count")],
+                description=f"{desc.description} (count)",
+            )
+            meter.create_observable_counter(
+                f"{desc.name}_sum",
+                callbacks=[field_callback(desc.name, "sum")],
+                unit=unit,
+                description=f"{desc.description} (sum)",
+            )
+
+    _INSTRUMENTED = True
+    return True
diff --git a/python/python/lance/tf/__init__.py b/python/python/lance/tf/__init__.py
deleted file mode 100644
index 1aa41beb99c..00000000000
--- a/python/python/lance/tf/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-import importlib.util
-
-if importlib.util.find_spec("tensorflow") is None:
-    raise ImportError(
-        "Tensorflow is not installed. Please install tensorflow"
-        + " to use lance.tf module.",
-    )
diff --git a/python/python/lance/tf/data.py b/python/python/lance/tf/data.py
deleted file mode 100644
index 68280f9211f..00000000000
--- a/python/python/lance/tf/data.py
+++ /dev/null
@@ -1,410 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-
-"""Tensorflow Dataset (`tf.data <https://www.tensorflow.org/guide/data>`_)
-implementation for Lance.
-
-.. warning::
-
-    Experimental feature. API stability is not guaranteed.
-"""
-
-from __future__ import annotations
-
-from functools import partial
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union
-
-import pyarrow as pa
-
-import lance
-from lance import LanceDataset
-from lance.arrow import EncodedImageType, FixedShapeImageTensorType, ImageURIType
-from lance.dependencies import _check_for_numpy
-from lance.dependencies import numpy as np
-from lance.dependencies import tensorflow as tf
-from lance.fragment import FragmentMetadata, LanceFragment
-from lance.log import LOGGER
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from lance import LanceNamespace
-
-
-def arrow_data_type_to_tf(dt: pa.DataType) -> tf.DType:
-    """Convert Pyarrow DataType to Tensorflow."""
-    if pa.types.is_boolean(dt):
-        return tf.bool
-    elif pa.types.is_int8(dt):
-        return tf.int8
-    elif pa.types.is_int16(dt):
-        return tf.int16
-    elif pa.types.is_int32(dt):
-        return tf.int32
-    elif pa.types.is_int64(dt):
-        return tf.int64
-    elif pa.types.is_uint8(dt):
-        return tf.uint8
-    elif pa.types.is_uint16(dt):
-        return tf.uint16
-    elif pa.types.is_uint32(dt):
-        return tf.uint32
-    elif pa.types.is_uint64(dt):
-        return tf.uint64
-    elif pa.types.is_float16(dt):
-        return tf.float16
-    elif pa.types.is_float32(dt):
-        return tf.float32
-    elif pa.types.is_float64(dt):
-        return tf.float64
-    elif (
-        pa.types.is_string(dt)
-        or pa.types.is_large_string(dt)
-        or pa.types.is_binary(dt)
-        or pa.types.is_large_binary(dt)
-    ):
-        return tf.string
-
-    raise TypeError(f"Arrow/Tf conversion: Unsupported arrow data type: {dt}")
-
-
-def data_type_to_tensor_spec(dt: pa.DataType) -> tf.TensorSpec:
-    """Convert PyArrow DataType to Tensorflow TensorSpec."""
-    if (
-        pa.types.is_boolean(dt)
-        or pa.types.is_integer(dt)
-        or pa.types.is_floating(dt)
-        or pa.types.is_string(dt)
-        or pa.types.is_binary(dt)
-    ):
-        return tf.TensorSpec(shape=(None,), dtype=arrow_data_type_to_tf(dt))
-    elif isinstance(dt, pa.FixedShapeTensorType):
-        return tf.TensorSpec(
-            shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.value_type)
-        )
-    elif pa.types.is_fixed_size_list(dt):
-        return tf.TensorSpec(
-            shape=(None, dt.list_size), dtype=arrow_data_type_to_tf(dt.value_type)
-        )
-    elif pa.types.is_list(dt) or pa.types.is_large_list(dt):
-        return tf.TensorSpec(
-            shape=(
-                None,
-                None,
-            ),
-            dtype=arrow_data_type_to_tf(dt.value_type),
-        )
-    elif pa.types.is_struct(dt):
-        return {field.name: data_type_to_tensor_spec(field.type) for field in dt}
-    elif isinstance(dt, (EncodedImageType, ImageURIType)):
-        return tf.TensorSpec(shape=(None,), dtype=tf.string)
-    elif isinstance(dt, FixedShapeImageTensorType):
-        return tf.TensorSpec(
-            shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.arrow_type)
-        )
-
-    raise TypeError("Unsupported data type: ", dt)
-
-
-def schema_to_spec(schema: pa.Schema) -> tf.TypeSpec:
-    """Convert PyArrow Schema to Tensorflow output signature."""
-    signature = {}
-    for name in schema.names:
-        field = schema.field(name)
-        signature[name] = data_type_to_tensor_spec(field.type)
-    return signature
-
-
-def column_to_tensor(array: pa.Array, tensor_spec: tf.TensorSpec) -> tf.Tensor:
-    """Convert a PyArrow array into a TensorFlow tensor."""
-    if isinstance(tensor_spec, tf.RaggedTensorSpec):
-        return tf.ragged.constant(array.to_pylist(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, pa.FixedShapeTensorType):
-        return tf.constant(array.to_numpy_ndarray(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, FixedShapeImageTensorType):
-        return tf.constant(array.to_numpy(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, pa.StructType):
-        return {
-            field.name: column_to_tensor(array.field(i), tensor_spec[field.name])
-            for (i, field) in enumerate(array.type)
-        }
-    else:
-        return tf.constant(array.to_pylist(), dtype=tensor_spec.dtype)
-
-
-def from_lance(
-    dataset: Optional[Union[str, Path, LanceDataset]] = None,
-    *,
-    columns: Optional[Union[List[str], Dict[str, str]]] = None,
-    batch_size: int = 256,
-    filter: Optional[str] = None,
-    fragments: Union[Iterable[int], Iterable[LanceFragment], tf.data.Dataset] = None,
-    output_signature: Optional[Dict[str, tf.TypeSpec]] = None,
-    namespace_client: Optional["LanceNamespace"] = None,
-    table_id: Optional[List[str]] = None,
-    ignore_namespace_table_storage_options: bool = False,
-) -> tf.data.Dataset:
-    """Create a ``tf.data.Dataset`` from a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset], optional
-        Lance dataset or dataset URI/path. Either ``dataset`` or both
-        ``namespace_client`` and ``table_id`` must be provided.
-    columns : Optional[List[str]], optional
-        List of columns to include in the output dataset.
-        If not set, all columns will be read.
-    batch_size : int, optional
-        Batch size, by default 256
-    filter : Optional[str], optional
-        SQL filter expression, by default None.
-    fragments : Union[List[LanceFragment], tf.data.Dataset], optional
-        If provided, only the fragments are read. It can be used to feed
-        for distributed training.
-    output_signature : Optional[tf.TypeSpec], optional
-        Override output signature of the returned tensors. If not provided,
-        the output signature is inferred from the projection Schema.
-    namespace_client : Optional[LanceNamespace], optional
-        Namespace client to resolve the table location when ``table_id`` is
-        provided.
-    table_id : Optional[List[str]], optional
-        Table identifier used together with ``namespace_client`` to locate
-        the table.
-    ignore_namespace_table_storage_options : bool, default False
-        When using ``namespace_client``/``table_id``, ignore storage options
-        returned by the namespace.
-
-    Examples
-    --------
-
-    .. code-block:: python
-
-        import tensorflow as tf
-        import lance.tf.data
-
-        ds = lance.tf.data.from_lance(
-            "s3://bucket/path",
-            columns=["image", "id"],
-            filter="catalog = 'train' AND split = 'train'",
-            batch_size=100)
-
-        for batch in ds.repeat(10).shuffle(128).map(io_decode):
-            print(batch["image"].shape)
-
-    ``from_lance`` can take an iterator or ``tf.data.Dataset`` of
-    Fragments. So that it can be used to feed for distributed training.
-
-    .. code-block:: python
-
-        import tensorflow as tf
-        import lance.tf.data
-
-        seed = 200  # seed to shuffle the fragments in distributed machines.
-        fragments = lance.tf.data.lance_fragments("s3://bucket/path")
-            repeat(10).shuffle(4, seed=seed)
-        ds = lance.tf.data.from_lance(
-            "s3://bucket/path",
-            columns=["image", "id"],
-            filter="catalog = 'train' AND split = 'train'",
-            fragments=fragments,
-            batch_size=100)
-        for batch in ds.shuffle(128).map(io_decode):
-            print(batch["image"].shape)
-
-    """
-    if isinstance(dataset, LanceDataset):
-        if namespace_client is not None or table_id is not None:
-            raise ValueError(
-                "Cannot specify 'namespace_client' or 'table_id' when passing "
-                "a LanceDataset instance"
-            )
-    else:
-        dataset = lance.dataset(
-            dataset,
-            namespace_client=namespace_client,
-            table_id=table_id,
-            ignore_namespace_table_storage_options=ignore_namespace_table_storage_options,
-        )
-
-    if isinstance(fragments, tf.data.Dataset):
-        fragments = list(fragments.as_numpy_iterator())
-    elif _check_for_numpy(fragments) and isinstance(fragments, np.ndarray):
-        fragments = list(fragments)
-
-    if fragments is not None:
-
-        def gen_fragments(fragments):
-            for f in fragments:
-                if isinstance(f, int) or (
-                    _check_for_numpy(f) and isinstance(f, np.integer)
-                ):
-                    yield LanceFragment(dataset, int(f))
-                elif isinstance(f, FragmentMetadata):
-                    yield LanceFragment(dataset, f.id)
-                elif isinstance(f, LanceFragment):
-                    yield f
-                else:
-                    raise TypeError(f"Invalid type passed to `fragments`: {type(f)}")
-
-        # A Generator of Fragments
-        fragments = gen_fragments(fragments)
-
-    scanner = dataset.scanner(
-        filter=filter, columns=columns, batch_size=batch_size, fragments=fragments
-    )
-
-    if output_signature is None:
-        schema = scanner.projected_schema
-        output_signature = schema_to_spec(schema)
-    LOGGER.debug("Output signature: %s", output_signature)
-
-    def generator():
-        for batch in scanner.to_batches():
-            yield {
-                name: column_to_tensor(batch[name], output_signature[name])
-                for name in batch.schema.names
-            }
-
-    return tf.data.Dataset.from_generator(generator, output_signature=output_signature)
-
-
-def lance_fragments(dataset: Union[str, Path, LanceDataset]) -> tf.data.Dataset:
-    """Create a ``tf.data.Dataset`` of Lance Fragments in the dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-    return tf.data.Dataset.from_tensor_slices(
-        [f.fragment_id for f in dataset.get_fragments()]
-    )
-
-
-def _ith_batch(i: int, batch_size: int, total_size: int) -> Tuple[int, int]:
-    """
-    Get the start and end index of the ith batch.
-
-    This takes into account the total_size, the total number of rows in the dataset.
-    """
-    start = i * batch_size
-    end = tf.math.minimum(start + batch_size, total_size)
-    return (start, end)
-
-
-def from_lance_batches(
-    dataset: Union[str, Path, LanceDataset],
-    *,
-    shuffle: bool = False,
-    seed: Optional[int] = None,
-    batch_size: int = 1024,
-    skip: int = 0,
-) -> tf.data.Dataset:
-    """
-    Create a ``tf.data.Dataset`` of batch indices for a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    shuffle : bool, optional
-        Shuffle the batches, by default False
-    seed : Optional[int], optional
-        Random seed for shuffling, by default None
-    batch_size : int, optional
-        Batch size, by default 1024
-    skip : int, optional
-        Number of batches to skip.
-
-    Returns
-    -------
-    tf.data.Dataset
-        A tensorflow dataset of batch slice ranges. These can be passed to
-        :func:`lance_take_batches` to create a Tensorflow dataset of batches.
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-    num_rows = dataset.count_rows()
-    num_batches = (num_rows + batch_size - 1) // batch_size
-    indices = tf.data.Dataset.range(num_batches, dtype=tf.int64)
-    if shuffle:
-        indices = indices.shuffle(num_batches, seed=seed)
-    if skip > 0:
-        indices = indices.skip(skip)
-    return indices.map(partial(_ith_batch, batch_size=batch_size, total_size=num_rows))
-
-
-def lance_take_batches(
-    dataset: Union[str, Path, LanceDataset],
-    batch_ranges: Iterable[Tuple[int, int]],
-    *,
-    columns: Optional[List[str]] = None,
-    output_signature: Optional[Dict[str, tf.TypeSpec]] = None,
-    batch_readahead: int = 10,
-) -> tf.data.Dataset:
-    """
-    Create a ``tf.data.Dataset`` of batches from a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    batch_ranges : Iterable[Tuple[int, int]]
-        Iterable of batch indices.
-    columns : Optional[List[str]], optional
-        List of columns to include in the output dataset.
-        If not set, all columns will be read.
-    output_signature : Optional[tf.TypeSpec], optional
-        Override output signature of the returned tensors. If not provided,
-        the output signature is inferred from the projection Schema.
-    batch_readahead : int, default 10
-        The number of batches to read ahead in parallel.
-
-    Examples
-    --------
-    You can compose this with ``from_lance_batches`` to create a randomized Tensorflow
-    dataset. With ``from_lance_batches``, you can deterministically randomized the
-    batches by setting ``seed``.
-
-    .. code-block:: python
-
-        batch_iter = from_lance_batches(dataset, batch_size=100, shuffle=True, seed=200)
-        batch_iter = batch_iter.as_numpy_iterator()
-        lance_ds = lance_take_batches(dataset, batch_iter)
-        lance_ds = lance_ds.unbatch().shuffle(500, seed=42).batch(100)
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-
-    if output_signature is None:
-        schema = dataset.scanner(columns=columns).projected_schema
-        output_signature = schema_to_spec(schema)
-    LOGGER.debug("Output signature: %s", output_signature)
-
-    def gen_ranges():
-        for start, end in batch_ranges:
-            yield (start, end)
-
-    def gen_batches():
-        batches = dataset._ds.take_scan(
-            gen_ranges(),
-            columns=columns,
-            batch_readahead=batch_readahead,
-        )
-        for batch in batches:
-            yield {
-                name: column_to_tensor(batch[name], output_signature[name])
-                for name in batch.schema.names
-            }
-
-    return tf.data.Dataset.from_generator(
-        gen_batches, output_signature=output_signature
-    )
-
-
-# Register `from_lance` to ``tf.data.Dataset``.
-tf.data.Dataset.from_lance = from_lance
-tf.data.Dataset.from_lance_batches = from_lance_batches
diff --git a/python/python/lance/types.py b/python/python/lance/types.py
index 41cc191e4d6..0ed2ae9f95e 100644
--- a/python/python/lance/types.py
+++ b/python/python/lance/types.py
@@ -52,6 +52,34 @@ def _casting_recordbatch_iter(
         yield batch
 
 
+def _is_materialized(data_obj: ReaderLike) -> bool:
+    """Whether ``data_obj`` is fully materialized in memory.
+
+    Materialized sources (tables, in-memory frames) can be wrapped in an
+    in-memory table for replay without spilling and to expose exact statistics.
+    Streaming or re-readable sources (readers, scanners, datasets, generators)
+    are not considered materialized.
+    """
+    if _check_for_pandas(data_obj) and isinstance(data_obj, pd.DataFrame):
+        return True
+    if isinstance(data_obj, (pa.Table, pa.RecordBatch)):
+        return True
+    if (
+        type(data_obj).__module__.startswith("polars")
+        and data_obj.__class__.__name__ == "DataFrame"
+    ):
+        return True
+    if isinstance(data_obj, dict):
+        return True
+    if (
+        isinstance(data_obj, list)
+        and len(data_obj) > 0
+        and isinstance(data_obj[0], dict)
+    ):
+        return True
+    return False
+
+
 def _coerce_reader(
     data_obj: ReaderLike, schema: Optional[pa.Schema] = None
 ) -> pa.RecordBatchReader:
diff --git a/python/python/lance/util.py b/python/python/lance/util.py
index 2161c4e0d45..180e2441b43 100644
--- a/python/python/lance/util.py
+++ b/python/python/lance/util.py
@@ -3,8 +3,18 @@
 
 from __future__ import annotations
 
+import uuid
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Iterator, Literal, Optional, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Union,
+    cast,
+)
 
 import pyarrow as pa
 
@@ -51,6 +61,29 @@ def td_to_micros(td: timedelta) -> int:
     return round(td / timedelta(microseconds=1))
 
 
+def _normalize_index_segment_ids(
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]],
+) -> Optional[List[str]]:
+    """Normalize a physical index segment selection to a list of UUID strings."""
+    if index_segments is None:
+        return None
+    if isinstance(index_segments, (str, uuid.UUID)):
+        raise TypeError(
+            "index_segments must be an iterable of str or uuid.UUID, "
+            f"not a single {type(index_segments)}."
+        )
+    segment_ids = []
+    for segment_id in index_segments:
+        if isinstance(segment_id, (str, uuid.UUID)):
+            segment_ids.append(str(segment_id))
+        else:
+            raise TypeError(
+                "index_segments must be an iterable of str or uuid.UUID. "
+                f"Got {type(segment_id)} instead."
+            )
+    return segment_ids
+
+
 class KMeans:
     """KMean model for clustering.
 
diff --git a/python/python/lance/vector.py b/python/python/lance/vector.py
index 5ce5e8b61e5..75e6b5ba15b 100644
--- a/python/python/lance/vector.py
+++ b/python/python/lance/vector.py
@@ -19,9 +19,10 @@
 )
 from .dependencies import numpy as np
 from .log import LOGGER
-from .util import MetricType, _normalize_metric_type
+from .util import MetricType, _normalize_index_segment_ids, _normalize_metric_type
 
 if TYPE_CHECKING:
+    import uuid
     from pathlib import Path
 
     from . import LanceDataset
@@ -761,13 +762,17 @@ def hamming_clustering_for_ivf_partition(
     index_name: str,
     partition_id: int,
     hamming_threshold: int,
+    *,
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
 ) -> pa.RecordBatchReader:
     """
     Perform hamming clustering on a partition of an IVF_FLAT index.
 
-    Loads a partition from an IVF_FLAT index on a hash column, computes
-    pairwise hamming distances between all hashes in the partition,
-    filters by threshold, and clusters the results using union-find.
+    Loads a partition from every segment of an IVF_FLAT index on a hash
+    column, computes pairwise hamming distances between all hashes in the
+    combined partition, filters by threshold, and clusters the results using
+    union-find. All segments of the logical index must share the same global
+    IVF centroids; an error is raised if they do not.
 
     Parameters
     ----------
@@ -779,6 +784,11 @@ def hamming_clustering_for_ivf_partition(
         The partition ID within the IVF_FLAT index
     hamming_threshold : int
         Maximum hamming distance to consider as similar
+    index_segments : iterable of str or uuid.UUID, optional
+        If specified, only these physical index segment UUIDs of the named
+        logical index contribute rows. Use
+        :meth:`LanceDataset.describe_indices` to obtain segment UUIDs from
+        ``IndexDescription.segments``. Defaults to all segments.
 
     Returns
     -------
@@ -789,30 +799,43 @@ def hamming_clustering_for_ivf_partition(
         - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
     """
     return dataset._ds.hamming_clustering_for_ivf_partition(
-        index_name, partition_id, hamming_threshold
+        index_name,
+        partition_id,
+        hamming_threshold,
+        _normalize_index_segment_ids(index_segments),
     )
 
 
 def get_ivf_partition_info(
     dataset: "LanceDataset",
     index_name: str,
+    *,
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
 ) -> List[dict]:
     """
     Get partition information for an IVF_FLAT index.
 
+    Partition sizes are aggregated across all segments of the logical index
+    unless a subset is selected via ``index_segments``.
+
     Parameters
     ----------
     dataset : LanceDataset
         The Lance dataset containing the hash column with an IVF_FLAT index.
     index_name : str
         Name of the IVF_FLAT index
+    index_segments : iterable of str or uuid.UUID, optional
+        If specified, only these physical index segment UUIDs of the named
+        logical index contribute to the sizes. Defaults to all segments.
 
     Returns
     -------
     list[dict]
         List of partition info dicts with 'partition_id' and 'size'
     """
-    return dataset._ds.get_ivf_partition_info(index_name)
+    return dataset._ds.get_ivf_partition_info(
+        index_name, _normalize_index_segment_ids(index_segments)
+    )
 
 
 def hamming_clustering_for_sample(
@@ -833,7 +856,8 @@ def hamming_clustering_for_sample(
     dataset : LanceDataset
         The Lance dataset containing the hash column.
     column : str
-        Name of the hash column (must be FixedSizeList<UInt8, 8>)
+        Name of the hash column (must be FixedSizeList<UInt8, N> where N is a
+        positive multiple of 8 bytes)
     sample_size : int, optional
         Number of rows to sample. If None, uses all rows.
     hamming_threshold : int, default 10
@@ -875,7 +899,8 @@ def hamming_clustering_for_range(
     dataset : LanceDataset
         The Lance dataset containing the hash column.
     column : str
-        Name of the hash column (must be FixedSizeList<UInt8, 8>)
+        Name of the hash column (must be FixedSizeList<UInt8, N> where N is a
+        positive multiple of 8 bytes)
     fragment_id : int
         The fragment ID to read from
     start_row : int
diff --git a/python/python/tests/compat/test_scalar_indices.py b/python/python/tests/compat/test_scalar_indices.py
index c3bf301eee0..4f54e9d31c2 100644
--- a/python/python/tests/compat/test_scalar_indices.py
+++ b/python/python/tests/compat/test_scalar_indices.py
@@ -9,6 +9,7 @@
 and written by other versions.
 """
 
+import os
 import shutil
 from pathlib import Path
 
@@ -320,9 +321,12 @@ def create(self):
             max_rows_per_file=100,
             data_storage_version=safe_data_storage_version(self.compat_version),
         )
-        dataset.create_scalar_index(
-            "text", "INVERTED", with_position=True, format_version=1
-        )
+        kwargs = {"with_position": True}
+        # Downgrade reads use older wheels, so current-created FTS indexes must
+        # stay on the legacy posting block layout.
+        if os.environ.get("LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE") == "1":
+            kwargs["block_size"] = 128
+        dataset.create_scalar_index("text", "INVERTED", format_version=1, **kwargs)
 
     def check_read(self):
         """Verify FTS index can be queried."""
@@ -351,6 +355,16 @@ def check_write(self):
     def skip_downgrade(self, version: str) -> bool:
         return version.startswith("0.")
 
+    def current_env(self, method_name: str) -> dict[str, str]:
+        if method_name == "create":
+            return {
+                "LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE": "1",
+                "LANCE_FTS_FORMAT_VERSION": "1",
+            }
+        if method_name == "check_write":
+            return {"LANCE_FTS_FORMAT_VERSION": "2"}
+        return {}
+
     def compat_env(self, version: str, method_name: str) -> dict[str, str]:
         if method_name in {"create", "check_write"}:
             return {"LANCE_FTS_FORMAT_VERSION": "1"}
diff --git a/python/python/tests/compat/venv_manager.py b/python/python/tests/compat/venv_manager.py
index c4b23486cd3..9cc09fc8f85 100644
--- a/python/python/tests/compat/venv_manager.py
+++ b/python/python/tests/compat/venv_manager.py
@@ -20,6 +20,7 @@
 from pathlib import Path
 from typing import Any, Optional
 
+import pytest
 from packaging.version import InvalidVersion, Version
 
 try:
@@ -96,6 +97,36 @@ def _safe(ref: str) -> str:
     return re.sub(r"[^A-Za-z0-9._-]", "_", ref)
 
 
+# Explicit skip whitelist for known (lance_version, python_version) incompatibilities.
+# Each entry is (max_lance_version_inclusive, min_python_version_inclusive).
+# A test is skipped when the tested lance version <= max_lance AND the current
+# Python >= min_python.  Add entries here only after verifying the incompatibility
+# is a runtime/ABI issue rather than a format regression — anything NOT listed
+# will surface as a test failure so new problems are visible.
+_COMPAT_SKIP: list[tuple[str, tuple[int, int]]] = [
+    # lance 0.22.x abi3 wheel was built with old PyO3 (<0.23) that crashes on
+    # Python 3.14+ due to removed internal CPython APIs.
+    ("0.22.0", (3, 14)),
+]
+
+
+def _skip_reason(lance_version: str) -> Optional[str]:
+    """Return a skip reason if this lance/Python combo is whitelisted, else None."""
+    try:
+        ver = Version(lance_version)
+    except InvalidVersion:
+        return None
+    py = sys.version_info[:2]
+    for max_lance, min_python in _COMPAT_SKIP:
+        if ver <= Version(max_lance) and py >= min_python:
+            py_str = f"{py[0]}.{py[1]}"
+            return (
+                f"Lance {lance_version} + Python {py_str}: whitelisted skip "
+                f"(see _COMPAT_SKIP in venv_manager.py)"
+            )
+    return None
+
+
 class VenvExecutor:
     """Manages a virtual environment with a specific Lance version."""
 
@@ -130,15 +161,29 @@ def python_path(self) -> Path:
     def _marker_path(self) -> Path:
         return self.venv_path / ".compat_ref"
 
+    @staticmethod
+    def _python_version_tag() -> str:
+        return f"{sys.version_info.major}.{sys.version_info.minor}"
+
     def _validate_venv(self) -> bool:
-        """A cached venv is reusable if it exists and its recorded ref matches. A marker
-        file is used (not `pip show`) so source-built commit refs also validate."""
+        """A cached venv is reusable if it exists, its recorded ref matches, and it was
+        built with the same Python major.minor as the current interpreter.
+
+        The marker file format is two lines: `<lance_ref>\\n<python_major.minor>`.
+        Old single-line markers (no Python version) are treated as stale so the venv
+        is rebuilt — this handles cached venvs from a different Python installation."""
         if not self.python_path.exists():
             return False
         try:
-            return self._marker_path.read_text().strip() == self.version
+            lines = self._marker_path.read_text().strip().splitlines()
         except OSError:
             return False
+        if not lines or lines[0] != self.version:
+            return False
+        # Require a Python version line; single-line markers are stale.
+        if len(lines) < 2 or lines[1] != self._python_version_tag():
+            return False
+        return True
 
     def create(self):
         """Create the virtual environment and install the specified Lance version."""
@@ -170,7 +215,9 @@ def create(self):
                     self._install_release_wheel()
                 else:
                     self._build_from_source()
-                self._marker_path.write_text(self.version)
+                self._marker_path.write_text(
+                    f"{self.version}\n{self._python_version_tag()}"
+                )
         self._created = True
 
     def _install_wheel(self, wheel: str):
@@ -264,10 +311,13 @@ def _ensure_subprocess(self):
         # Start persistent subprocess
         runner_script = Path(__file__).parent / "venv_runner.py"
 
-        # Set PYTHONPATH to include the tests directory
+        # Set PYTHONPATH so the subprocess can import compat test modules.
+        # pytest adds the `tests/` directory (the first ancestor without __init__.py)
+        # to sys.path, so test modules are imported as `compat.<module>`.
         env = os.environ.copy()
         tests_dir = Path(__file__).parent.parent
         env["PYTHONPATH"] = str(tests_dir)
+        env.setdefault("RUST_BACKTRACE", "full")
 
         # Capture stderr to a file so a Rust panic (which crashes the runner) can be
         # surfaced in the error instead of an opaque "broken pipe".
@@ -357,6 +407,10 @@ def execute_method(
         if not self._created:
             raise RuntimeError("Virtual environment not created. Call create() first.")
 
+        reason = _skip_reason(self.version)
+        if reason:
+            pytest.skip(reason)
+
         # Ensure subprocess is running
         self._ensure_subprocess()
         try:
@@ -379,14 +433,16 @@ def execute_method(
 
         except (BrokenPipeError, EOFError, struct.error) as e:
             # Subprocess died (usually a Rust panic); flush it, then surface that.
+            returncode = "unknown"
             if self._subprocess is not None:
                 try:
                     self._subprocess.wait(timeout=2)
+                    returncode = str(self._subprocess.returncode)
                 except Exception:
                     pass
             panic = self._last_panic()
             detail = panic or f"subprocess communication failed: {e}"
-            raise RuntimeError(f"Lance {self.version}: {detail}")
+            raise RuntimeError(f"Lance {self.version} (exit={returncode}): {detail}")
 
     def cleanup(self):
         """Remove the virtual environment directory and terminate subprocess."""
@@ -404,8 +460,6 @@ def cleanup(self):
 
         # Remove venv directory
         if self.venv_path.exists():
-            import shutil
-
             shutil.rmtree(self.venv_path)
         self._created = False
 
diff --git a/python/python/tests/test_arrow.py b/python/python/tests/test_arrow.py
index 92ab52021ff..a0be09846e1 100644
--- a/python/python/tests/test_arrow.py
+++ b/python/python/tests/test_arrow.py
@@ -273,8 +273,6 @@ def test_image_uri_arrays(tmp_path: Path, png_uris):
 
 
 def test_image_tensor_arrays(tmp_path: Path, png_uris):
-    tf = pytest.importorskip("tensorflow")
-
     n = 10
 
     encoded_image_array = ImageURIArray.from_uris(png_uris).read_uris()
@@ -297,22 +295,22 @@ def test_image_tensor_arrays(tmp_path: Path, png_uris):
     assert tensor_image_array.storage.type == pa.list_(pa.uint8(), 4)
     assert tensor_image_array[2].as_py() == [42, 42, 42, 255]
 
-    test_tensor = tf.constant(
-        np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4))
-    )
+    test_tensor = np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4))
 
     assert test_tensor.shape == (n, 1, 1, 4)
-    assert tf.math.reduce_all(
-        tf.convert_to_tensor(tensor_image_array.to_numpy()) == test_tensor
-    )
+    assert np.array_equal(tensor_image_array.to_numpy(), test_tensor)
     assert tensor_image_array.to_encoded().to_tensor() == tensor_image_array
 
     def png_encoder(images):
-        import tensorflow as tf
+        import io
 
-        encoded_images = (
-            tf.io.encode_png(x).numpy() for x in tf.convert_to_tensor(images)
-        )
+        from PIL import Image  # pyright: ignore[reportMissingImports]
+
+        encoded_images = []
+        for image in images:
+            with io.BytesIO() as buf:
+                Image.fromarray(image).save(buf, format="PNG")
+                encoded_images.append(buf.getvalue())
         return pa.array(encoded_images, type=pa.binary())
 
     assert tensor_image_array.to_encoded(png_encoder).to_tensor() == tensor_image_array
@@ -324,20 +322,18 @@ def png_encoder(images):
         uris = [str(Path(x)) for x in uris]
 
     encoded_image_array = ImageArray.from_array(uris).read_uris()
-    with pytest.raises(
-        tf.errors.InvalidArgumentError, match="Shapes of all inputs must match"
-    ):
+    with pytest.raises(ValueError, match="all input arrays must have the same shape"):
         encoded_image_array.to_tensor()
 
     pattern = r"(object at) 0x[\w\d]+(:?>)"
     repl = r"\1 0x..\2"
-    assert re.sub(pattern, repl, encoded_image_array.__repr__()) == (
-        "<lance.arrow.EncodedImageArray object at 0x..>\n"
-        "[<tf.Tensor: shape=(1, 1, 4), dtype=uint8, numpy=array([[[ 42,  42,  42, "
-        "255]]], dtype=uint8)>, ..]\n"
+    repr_ = re.sub(pattern, repl, encoded_image_array.__repr__())
+    assert repr_.startswith("<lance.arrow.EncodedImageArray object at 0x..>\n[<PIL.")
+    assert "image mode=RGBA size=1x1" in repr_
+    assert (
         "b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\x01\\x00\\x00"
         "\\x00\\x01\\x08\\x06\\x00\\x00\\x00\\x1f'"
-    )
+    ) in repr_
 
 
 def test_roundtrip_image_tensor(tmp_path: Path):
diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py
index d7de9b43340..5d9359587cb 100644
--- a/python/python/tests/test_blob.py
+++ b/python/python/tests/test_blob.py
@@ -1,12 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+import gc
 import importlib
 import io
+import queue
 import subprocess
 import sys
 import tarfile
 import textwrap
+import threading
 import uuid
 from pathlib import Path
 
@@ -271,6 +274,44 @@ def dataset_with_blobs(tmp_path):
     return ds
 
 
+@pytest.fixture
+def dataset_with_mixed_blob_v2(tmp_path):
+    external_blob = tmp_path / "external.bin"
+    external_blob.write_bytes(b"external")
+    payloads = [b"in", b"packed!!", b"dedicated payload", b"external", None, b""]
+    values = [
+        payloads[0],
+        payloads[1],
+        payloads[2],
+        external_blob.as_uri(),
+        None,
+        payloads[5],
+    ]
+    schema = pa.schema(
+        [
+            lance.blob_field(
+                "blobs",
+                inline_size_threshold=4,
+                dedicated_size_threshold=12,
+            ),
+            pa.field("idx", pa.uint64()),
+        ]
+    )
+    table = pa.table(
+        {"blobs": lance.blob_array(values), "idx": range(len(values))}, schema=schema
+    )
+    dataset = lance.write_dataset(
+        table,
+        tmp_path / "mixed_blob_v2",
+        data_storage_version="2.2",
+        allow_external_blob_outside_bases=True,
+        max_rows_per_file=2,
+        max_rows_per_group=2,
+    )
+    assert len(dataset.get_fragments()) == 3
+    return dataset, payloads
+
+
 def test_blob_files(dataset_with_blobs):
     row_ids = (
         dataset_with_blobs.to_table(columns=[], with_row_id=True)
@@ -514,6 +555,172 @@ def test_read_blobs_without_preserve_order_returns_same_rows(
     assert sorted(blobs) == sorted(expected)
 
 
+@pytest.mark.parametrize("selection_kind", ["ids", "addresses", "indices"])
+@pytest.mark.parametrize(
+    "selected_indices",
+    [
+        [0, 5],
+        [0, 5, 4],
+        [5],
+        [0, 5, 1],
+        [4, 4, 5],
+    ],
+)
+def test_blob_selection_apis_preserve_nulls_and_empty_values(
+    dataset_with_mixed_blob_v2, selection_kind, selected_indices
+):
+    dataset, payloads = dataset_with_mixed_blob_v2
+    if selection_kind == "ids":
+        all_selectors = _blob_row_ids(dataset)
+    elif selection_kind == "addresses":
+        all_selectors = _blob_row_addresses(dataset)
+    else:
+        all_selectors = list(range(len(payloads)))
+    selectors = [all_selectors[index] for index in selected_indices]
+    kwargs = {selection_kind: selectors}
+    expected = [payloads[index] for index in selected_indices]
+
+    blob_files = dataset.take_blobs("blobs", **kwargs)
+    assert [None if blob is None else blob.readall() for blob in blob_files] == expected
+
+    blobs = dataset.read_blobs("blobs", **kwargs, preserve_order=True)
+    assert [data for _, data in blobs] == expected
+
+
+@pytest.mark.parametrize("selection_kind", ["ids", "addresses", "indices"])
+def test_read_blob_ranges_mixed_sources_preserves_request_identity(
+    dataset_with_mixed_blob_v2, selection_kind
+):
+    dataset, payloads = dataset_with_mixed_blob_v2
+    selected_indices = [3, 1, 1, 0, 2, 4, 5]
+    ranges = [(1, 3), (2, 4), (0, 4), (1, 1), (5, 0), (0, 0), (0, 0)]
+    addresses = _blob_row_addresses(dataset)
+    if selection_kind == "ids":
+        all_selectors = _blob_row_ids(dataset)
+    elif selection_kind == "addresses":
+        all_selectors = addresses
+    else:
+        all_selectors = list(range(len(payloads)))
+    selectors = [all_selectors[index] for index in selected_indices]
+    requests = [
+        (row, offset, length) for row, (offset, length) in zip(selectors, ranges)
+    ]
+
+    results = dataset.read_blob_ranges(
+        "blobs",
+        requests,
+        selector=selection_kind,
+        io_buffer_size=1024,
+        preserve_order=True,
+    )
+
+    expected = []
+    for request_index, (row_index, (offset, length)) in enumerate(
+        zip(selected_indices, ranges)
+    ):
+        payload = payloads[row_index]
+        expected.append(
+            (
+                request_index,
+                addresses[row_index],
+                None if payload is None else payload[offset : offset + length],
+            )
+        )
+    assert results == expected
+
+
+def test_blob_selection_apis_preserve_blob_v1_nulls_and_empty_values(tmp_path):
+    schema = pa.schema(
+        [
+            pa.field(
+                "blobs",
+                pa.large_binary(),
+                metadata={"lance-encoding:blob": "true"},
+            ),
+            pa.field("idx", pa.uint64()),
+        ]
+    )
+    dataset = lance.write_dataset(
+        pa.table({"blobs": [None, b"", b"abc"], "idx": range(3)}, schema=schema),
+        tmp_path / "blob_v1_nulls",
+        data_storage_version="2.0",
+    )
+    addresses = _blob_row_addresses(dataset)
+
+    blob_files = dataset.take_blobs("blobs", indices=[0, 1, 2])
+    assert [None if blob is None else blob.readall() for blob in blob_files] == [
+        None,
+        b"",
+        b"abc",
+    ]
+
+    blobs = dataset.read_blobs("blobs", indices=[0, 1, 2])
+    assert [data for _, data in blobs] == [None, b"", b"abc"]
+
+    results = dataset.read_blob_ranges(
+        "blobs",
+        [(0, 0, 0), (0, 0, 1), (1, 0, 0), (2, 1, 1)],
+        selector="indices",
+    )
+
+    assert results == [
+        (0, addresses[0], None),
+        (1, addresses[0], None),
+        (2, addresses[1], b""),
+        (3, addresses[2], b"b"),
+    ]
+
+
+def test_read_blob_ranges_without_preserve_order_keeps_request_identity(
+    dataset_with_mixed_blob_v2,
+):
+    dataset, _ = dataset_with_mixed_blob_v2
+    results = dataset.read_blob_ranges(
+        "blobs",
+        [(3, 0, 1), (4, 0, 1), (1, 1, 2), (0, 0, 2)],
+        selector="indices",
+        preserve_order=False,
+    )
+
+    assert sorted(results) == sorted(
+        [
+            (0, _blob_row_addresses(dataset)[3], b"e"),
+            (1, _blob_row_addresses(dataset)[4], None),
+            (2, _blob_row_addresses(dataset)[1], b"ac"),
+            (3, _blob_row_addresses(dataset)[0], b"in"),
+        ]
+    )
+
+
+@pytest.mark.parametrize(
+    ("requests", "message"),
+    [
+        ([(0, 2**64 - 1, 2)], "offset \\+ length overflowed"),
+        ([(0, 0, 4)], "exceeds blob size"),
+    ],
+)
+def test_read_blob_ranges_rejects_invalid_ranges(
+    dataset_with_mixed_blob_v2, requests, message
+):
+    dataset, _ = dataset_with_mixed_blob_v2
+
+    with pytest.raises(ValueError, match=message):
+        dataset.read_blob_ranges("blobs", requests, selector="indices")
+
+
+def test_read_blob_ranges_rejects_invalid_selector(dataset_with_mixed_blob_v2):
+    dataset, _ = dataset_with_mixed_blob_v2
+
+    with pytest.raises(ValueError, match="selector must be one of"):
+        dataset.read_blob_ranges("blobs", [], selector="offsets")
+
+
+def test_read_blob_ranges_accepts_empty_batch(dataset_with_mixed_blob_v2):
+    dataset, _ = dataset_with_mixed_blob_v2
+
+    assert dataset.read_blob_ranges("blobs", [], selector="indices") == []
+
+
 def test_blob_file_seek(tmp_path, dataset_with_blobs):
     row_ids = (
         dataset_with_blobs.to_table(columns=[], with_row_id=True)
@@ -544,8 +751,7 @@ def test_null_blobs(tmp_path):
     ds = lance.write_dataset(table, tmp_path / "test_ds")
 
     blobs = ds.take_blobs("blob", ids=range(100))
-    for blob in blobs:
-        assert blob.size() == 0
+    assert blobs == [None] * 100
 
     ds.insert(pa.table({"id": pa.array(range(100, 200), pa.uint64())}))
 
@@ -559,8 +765,7 @@ def test_null_blobs(tmp_path):
 
     for blob_col in ["blob", "more_blob"]:
         blobs = ds.take_blobs(blob_col, indices=range(100, 200))
-        for blob in blobs:
-            assert blob.size() == 0
+        assert blobs == [None] * 100
 
         blobs = ds.to_table(columns=[blob_col])
         for blob in blobs.column(blob_col):
@@ -1054,7 +1259,14 @@ def test_blob_extension_add_columns_all_nulls_blob_v2(tmp_path):
     ds.add_columns(lance.blob_field("blob"))
 
     assert ds.to_table(columns=["blob"]).column("blob").to_pylist() == [None] * 4
-    assert ds.take_blobs("blob", indices=range(4)) == []
+    assert ds.take_blobs("blob", indices=range(4)) == [None] * 4
+    assert [data for _, data in ds.read_blobs("blob", indices=range(4))] == [None] * 4
+    ranges = ds.read_blob_ranges(
+        "blob",
+        [(index, 0, 1) for index in range(4)],
+        selector="indices",
+    )
+    assert [data for _, _, data in ranges] == [None] * 4
 
 
 def test_blob_descriptor_array_builder_writes_prepared_packed_blob_for_data_replacement(
@@ -1119,6 +1331,304 @@ def test_blob_descriptor_array_builder_writes_prepared_packed_blob_for_data_repl
     assert blobs[0].readall() == b"replacement"
 
 
+@pytest.mark.parametrize(
+    "payload",
+    [
+        pytest.param(b"payload", id="bytes"),
+        pytest.param(bytearray(b"payload"), id="bytearray"),
+        pytest.param(memoryview(b"payload"), id="memoryview"),
+        pytest.param(list(b"payload"), id="integer_sequence"),
+    ],
+)
+def test_packed_blob_writer_scalar_buffer_inputs(tmp_path, payload):
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blob(payload)
+    descriptors = packed.finish()
+
+    assert [repr(descriptor) for descriptor in descriptors] == [
+        "Packed { blob_id: 7, offset: 0, size: 7 }"
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"payload"
+
+
+@pytest.mark.parametrize("array_type", [pa.binary(), pa.large_binary()])
+@pytest.mark.parametrize("as_chunked", [False, True], ids=["array", "chunked_array"])
+@pytest.mark.parametrize(
+    "values,slice_offset,slice_length,expected_values,expected_data",
+    [
+        pytest.param(
+            [b"prefix", b"a", None, b"", b"bc", b"suffix"],
+            1,
+            4,
+            [b"a", None, b"", b"bc"],
+            b"abc",
+            id="interleaved_null",
+        ),
+        pytest.param(
+            [b"prefix", b"a", b"", b"bc", b"suffix"],
+            1,
+            3,
+            [b"a", b"", b"bc"],
+            b"abc",
+            id="all_valid",
+        ),
+        pytest.param(
+            [b"prefix", None, None, b"suffix"],
+            1,
+            2,
+            [None, None],
+            b"",
+            id="all_null",
+        ),
+        pytest.param(
+            [b"prefix", b"suffix"],
+            1,
+            0,
+            [],
+            b"",
+            id="empty",
+        ),
+    ],
+)
+def test_packed_blob_writer_bulk_arrow_array(
+    tmp_path,
+    array_type,
+    as_chunked,
+    values,
+    slice_offset,
+    slice_length,
+    expected_values,
+    expected_data,
+):
+    file_id = str(uuid.uuid4())
+    data_file_name = f"{file_id}.lance"
+    blob_id = 7
+    payloads = pa.array(values, type=array_type).slice(slice_offset, slice_length)
+    if as_chunked:
+        split_at = max(1, len(payloads) // 2)
+        payloads = pa.chunked_array(
+            [payloads.slice(0, split_at), payloads.slice(split_at)]
+        )
+
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(data_file_name, blob_id)
+    with pytest.raises(ValueError, match="available after finish_array"):
+        packed.field
+    packed.write_blobs(payloads)
+    descriptors = packed.finish_array("image_bytes")
+    descriptor_field = packed.field
+
+    expected_descriptors = []
+    position = 0
+    for value in expected_values:
+        if value is None:
+            expected_descriptors.append(None)
+        else:
+            expected_descriptors.append(
+                {
+                    "kind": 1,
+                    "data": None,
+                    "uri": None,
+                    "blob_id": blob_id,
+                    "blob_size": len(value),
+                    "position": position,
+                }
+            )
+            position += len(value)
+
+    assert descriptors.to_pylist() == expected_descriptors
+    assert descriptor_field == lance.BlobDescriptorArrayBuilder("image_bytes").field
+    assert descriptor_field.metadata[b"ARROW:extension:name"] == b"lance.blob.v2"
+    assert pa.record_batch(
+        [descriptors], schema=pa.schema([descriptor_field])
+    ).num_rows == len(expected_values)
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == expected_data
+
+
+@pytest.mark.parametrize(
+    "array_type,offset_type",
+    [
+        pytest.param(pa.binary(), pa.int32(), id="binary"),
+        pytest.param(pa.large_binary(), pa.int64(), id="large_binary"),
+    ],
+)
+def test_packed_blob_writer_bulk_excludes_physical_null_bytes(
+    tmp_path, array_type, offset_type
+):
+    offsets = pa.array([0, 1, 5, 5, 7], type=offset_type).buffers()[1]
+    payloads = pa.Array.from_buffers(
+        array_type,
+        4,
+        [
+            pa.py_buffer(bytes([0b00001101])),
+            offsets,
+            pa.py_buffer(b"aJUNKbc"),
+        ],
+    )
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blobs(payloads)
+    descriptors = packed.finish_array("image_bytes")
+
+    assert descriptors.to_pylist() == [
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 1,
+            "position": 0,
+        },
+        None,
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 0,
+            "position": 1,
+        },
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 2,
+            "position": 1,
+        },
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"abc"
+
+
+@pytest.mark.parametrize(
+    "array_type,offset_type",
+    [
+        pytest.param(pa.binary(), pa.int32(), id="binary"),
+        pytest.param(pa.large_binary(), pa.int64(), id="large_binary"),
+    ],
+)
+@pytest.mark.parametrize("as_chunked", [False, True], ids=["array", "chunked_array"])
+def test_packed_blob_writer_bulk_rejects_non_monotonic_offsets(
+    tmp_path, array_type, offset_type, as_chunked
+):
+    offsets = pa.array([0, 2, 1, 2], type=offset_type).buffers()[1]
+    malformed = pa.Array.from_buffers(
+        array_type,
+        3,
+        [None, offsets, pa.py_buffer(b"ab")],
+    )
+    payloads = malformed
+    expected_context = "Packed blob payload array"
+    if as_chunked:
+        payloads = pa.chunked_array([pa.array([b"valid"], type=array_type), malformed])
+        expected_context = "Packed blob payload chunk 1"
+
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    with pytest.raises(ValueError, match="invalid Arrow data") as error:
+        packed.write_blobs(payloads)
+    assert expected_context in str(error.value)
+    assert "non-monotonic offset" in str(error.value)
+
+    packed.write_blob(b"still usable")
+    descriptors = packed.finish_array("blob")
+    assert len(descriptors) == 1
+    assert (
+        _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"still usable"
+    )
+
+
+def test_packed_blob_writer_mixed_calls_preserve_legacy_finish_alignment(tmp_path):
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blob(b"s")
+    packed.write_blobs(pa.array([b"a", None, b""], type=pa.binary()))
+    packed.write_blobs(pa.array([None, b"bc"], type=pa.large_binary()))
+    descriptors = packed.finish()
+
+    assert [repr(descriptor) for descriptor in descriptors] == [
+        "Packed { blob_id: 7, offset: 0, size: 1 }",
+        "Packed { blob_id: 7, offset: 1, size: 1 }",
+        "Null",
+        "Packed { blob_id: 7, offset: 2, size: 0 }",
+        "Null",
+        "Packed { blob_id: 7, offset: 2, size: 2 }",
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"sabc"
+
+
+@pytest.mark.parametrize(
+    "payloads",
+    [
+        pytest.param(pa.array([1, 2], type=pa.int32()), id="array"),
+        pytest.param(pa.chunked_array([[1], [2]], type=pa.int32()), id="chunked_array"),
+        pytest.param([b"not an Arrow array"], id="python_list"),
+    ],
+)
+def test_packed_blob_writer_bulk_rejects_non_binary_array(tmp_path, payloads):
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer("data-file.lance", 1)
+
+    with pytest.raises(ValueError, match="Binary") as error:
+        packed.write_blobs(payloads)
+    if isinstance(payloads, pa.Array):
+        assert "chunk" not in str(error.value)
+
+    packed.write_blob(b"still usable")
+    assert len(packed.finish_array("blob")) == 1
+
+
+@pytest.mark.parametrize(
+    "open_writer",
+    [
+        pytest.param("open_packed_blob_writer", id="packed"),
+        pytest.param("open_dedicated_blob_writer", id="dedicated"),
+    ],
+)
+def test_failed_blob_writer_traceback_can_be_released_on_another_thread(
+    tmp_path, monkeypatch, open_writer
+):
+    failures = queue.Queue()
+
+    def fail_with_live_writer():
+        files = LanceFileSession(tmp_path)
+        writer = getattr(files, open_writer)("data-file.lance", 1)
+        assert writer.blob_id == 1
+        try:
+            raise OSError("simulated object-store write failure")
+        except OSError as error:
+            # The traceback retains this frame and its local writer, matching a
+            # writer-thread failure handed to an owning thread for propagation.
+            failures.put(error)
+
+    writer_thread = threading.Thread(target=fail_with_live_writer)
+    writer_thread.start()
+    writer_thread.join(timeout=10)
+    assert not writer_thread.is_alive()
+
+    unraisable = []
+    monkeypatch.setattr(sys, "unraisablehook", unraisable.append)
+    error = failures.get_nowait()
+    assert str(error) == "simulated object-store write failure"
+    del error
+    gc.collect()
+
+    assert unraisable == []
+
+
 def test_blob_extension_write_fragments_external_denied_by_default(tmp_path):
     blob_path = tmp_path / "external_blob.bin"
 
@@ -1665,7 +2175,7 @@ def test_write_nested_blob_v2_and_take_by_field_path(tmp_path):
     with blobs[1] as f:
         assert f.read() == packed
 
-    assert dataset.take_blobs("info.blob", indices=[2]) == []
+    assert dataset.take_blobs("info.blob", indices=[2]) == [None]
 
 
 def test_to_pandas_returns_blob_files_for_projected_nested_fields(
diff --git a/python/python/tests/test_coerce_query_vector.py b/python/python/tests/test_coerce_query_vector.py
new file mode 100644
index 00000000000..7a5d341f8e0
--- /dev/null
+++ b/python/python/tests/test_coerce_query_vector.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Tests for _coerce_query_vector to ensure invalid input types
+raise TypeError instead of falling into the numpy conversion branch."""
+
+import numpy as np
+import pyarrow as pa
+import pytest
+from lance.dataset import _coerce_query_vector
+
+
+class TestCoerceQueryVectorInvalidTypes:
+    """Non-vector inputs should raise TypeError, not numpy ValueError."""
+
+    def test_string_raises_typeerror(self):
+        with pytest.raises(TypeError, match="query vector must be list-like"):
+            _coerce_query_vector("not a vector")
+
+    def test_integer_raises_typeerror(self):
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(42)
+
+    def test_object_raises_typeerror(self):
+        """A random object that is not array-like should raise TypeError."""
+
+        class NotAVector:
+            pass
+
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(NotAVector())
+
+    def test_none_raises_typeerror(self):
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(None)
+
+
+class TestCoerceQueryVectorValidTypes:
+    """Valid vector inputs should be coerced successfully."""
+
+    def test_list_of_floats(self):
+        result, dim = _coerce_query_vector([1.0, 2.0, 3.0])
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_tuple_of_floats(self):
+        result, dim = _coerce_query_vector((1.0, 2.0, 3.0))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_numpy_array(self):
+        result, dim = _coerce_query_vector(np.array([1.0, 2.0, 3.0]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_float_array(self):
+        result, dim = _coerce_query_vector(pa.array([1.0, 2.0, 3.0]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_int_array_cast_to_float(self):
+        result, dim = _coerce_query_vector(pa.array([1, 2, 3]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_chunked_array(self):
+        chunked = pa.chunked_array([[1.0, 2.0, 3.0]])
+        result, dim = _coerce_query_vector(chunked)
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py
index e449c6b9865..8891c397a0e 100644
--- a/python/python/tests/test_dataset.py
+++ b/python/python/tests/test_dataset.py
@@ -31,7 +31,7 @@
 from lance.commit import CommitConflictError
 from lance.dataset import LANCE_COMMIT_MESSAGE_KEY, AutoCleanupConfig
 from lance.debug import format_fragment
-from lance.file import stable_version
+from lance.file import LanceFileWriter, stable_version
 from lance.schema import LanceSchema
 from lance.util import validate_vector_index
 
@@ -2424,6 +2424,29 @@ def test_merge_insert(tmp_path: Path):
         check_merge_stats(merge_dict, (None, None, None))
 
 
+@pytest.mark.parametrize("materialized", [True, False])
+def test_merge_insert_input_kinds(tmp_path: Path, materialized: bool):
+    # A materialized pa.Table is routed through the in-memory (MemTable) path,
+    # while a RecordBatchReader is routed through the streaming path. Both must
+    # produce identical results.
+    schema = pa.schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
+    base = pa.table({"id": range(5), "value": [0] * 5}, schema=schema)
+    new = pa.table({"id": [1, 2, 5, 6], "value": [10, 20, 50, 60]}, schema=schema)
+
+    dataset = lance.write_dataset(base, tmp_path / "dataset", mode="create")
+    source = new if materialized else new.to_reader()
+
+    dataset.merge_insert(
+        "id"
+    ).when_matched_update_all().when_not_matched_insert_all().execute(source)
+
+    result = dataset.to_table().sort_by("id").to_pydict()
+    assert result == {
+        "id": [0, 1, 2, 3, 4, 5, 6],
+        "value": [0, 10, 20, 0, 0, 50, 60],
+    }
+
+
 def test_merge_insert_subcols(tmp_path: Path):
     initial_data = pa.table(
         {
@@ -2566,6 +2589,66 @@ def test_merge_insert_full_fragment_rewrite_json_e2e(tmp_path: Path):
     assert sample_result.num_rows == 3
 
 
+def test_merge_insert_subcols_with_json_column(tmp_path: Path):
+    """Test merge_insert with subschema update on a JSON extension type column.
+
+    Previously this would fail with:
+    'Incorrect datatype for StructArray field, expected Utf8 got LargeBinary'
+    because the update_fragments path didn't handle the Arrow JSON ↔ Lance JSON
+    type mismatch during interleave.
+    """
+    import json
+
+    json_type = pa.json_()
+    initial_data = pa.table(
+        {
+            "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()),
+            "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()),
+            "score": pa.array([10, 20, 30, 40, 50], type=pa.int64()),
+            "meta": pa.array(
+                ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'],
+                type=json_type,
+            ),
+        }
+    )
+    dataset = lance.write_dataset(initial_data, tmp_path / "merge_json_subcols")
+
+    # Subschema update: only provide id (key) + meta (JSON column to update)
+    new_values = pa.table(
+        {
+            "id": pa.array([2, 4], type=pa.int64()),
+            "meta": pa.array(
+                ['{"updated":true,"id":2}', '{"updated":true,"id":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    # This should NOT raise a type mismatch error
+    dataset.merge_insert("id").when_matched_update_all().execute(new_values)
+
+    # Verify results
+    result = dataset.to_table().sort_by("id")
+    ids = result.column("id").to_pylist()
+    scores = result.column("score").to_pylist()
+    metas = result.column("meta").to_pylist()
+
+    # Score column (not in update) should be preserved
+    assert scores == [10, 20, 30, 40, 50]
+
+    # Meta column should be updated for id=2 and id=4
+    for id_val, meta_val in zip(ids, metas):
+        parsed = json.loads(meta_val) if isinstance(meta_val, str) else meta_val
+        if id_val in (2, 4):
+            assert parsed.get("updated") is True, (
+                f"id={id_val} should have updated meta, got {meta_val}"
+            )
+        else:
+            assert "x" in str(parsed), (
+                f"id={id_val} should have original meta, got {meta_val}"
+            )
+
+
 def test_merge_insert_defaults_to_pk_when_on_omitted(tmp_path):
     base_dir = tmp_path / "merge_insert_pk_default"
 
@@ -4627,16 +4710,20 @@ def test_late_materialization_param(tmp_path: Path):
     )
     filt = "filter % 2 == 0"
 
-    assert "(values)" in dataset.scanner(
-        filter=filt, late_materialization=None
-    ).explain_plan(True)
+    # A late-materialized column is fetched by a row-stream read
+    # (`projection=[values], source=stream`); an eager column appears in the
+    # scan projection
+    late = "projection=[values], source=stream"
+    assert late in dataset.scanner(filter=filt, late_materialization=None).explain_plan(
+        True
+    )
     assert ", values" in dataset.scanner(
         filter=filt, late_materialization=False
     ).explain_plan(True)
-    assert "(values)" in dataset.scanner(
-        filter=filt, late_materialization=True
-    ).explain_plan(True)
-    assert "(values)" in dataset.scanner(
+    assert late in dataset.scanner(filter=filt, late_materialization=True).explain_plan(
+        True
+    )
+    assert late in dataset.scanner(
         filter=filt, late_materialization=["values"]
     ).explain_plan(True)
     assert ", values" in dataset.scanner(
@@ -4903,6 +4990,281 @@ def test_data_replacement(tmp_path: Path):
     assert tbl == expected
 
 
+def _write_overlay_file(
+    dataset, base_dir: Path, name: str, batch: pa.Table, fields: List[int]
+):
+    """Write an overlay value file (one value column per covered field, no key
+    column) and return a DataFile mapping its columns to the given dataset
+    `fields`. The file version is copied from a base data file."""
+    path = base_dir / "data" / name
+    with LanceFileWriter(str(path)) as writer:
+        writer.write_batch(batch)
+    base_df = dataset.get_fragments()[0].metadata.files[0]
+    return lance.fragment.DataFile(
+        path=name,
+        fields=fields,
+        column_indices=list(range(len(fields))),
+        file_major_version=base_df.file_major_version,
+        file_minor_version=base_df.file_minor_version,
+        file_size_bytes=os.path.getsize(path),
+    )
+
+
+def test_data_overlay_dense(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    # Overlay `val` at physical offsets {1, 4} with new values.
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    assert data_file.fields == [1]  # `val` is field id 1
+
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4])
+    op = lance.LanceOperation.DataOverlay(
+        [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+    )
+    dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version)
+
+    result = dataset.to_table()
+    assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90]
+    # The unrelated `id` column is untouched.
+    assert result.column("id").to_pylist() == list(range(10))
+
+
+def test_data_overlay_newest_wins(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    older = _write_overlay_file(
+        dataset,
+        base_dir,
+        "older.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [
+                lance.LanceOperation.DataOverlayGroup(
+                    0,
+                    [lance.LanceOperation.DataOverlayFile(older, offsets=[1, 4])],
+                )
+            ]
+        ),
+        read_version=dataset.version,
+    )
+    # A newer overlay re-covers offset 1; it must win there.
+    newer = _write_overlay_file(
+        dataset,
+        base_dir,
+        "newer.lance",
+        pa.table({"val": pa.array([999], pa.int32())}),
+        fields=[1],
+    )
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [
+                lance.LanceOperation.DataOverlayGroup(
+                    0, [lance.LanceOperation.DataOverlayFile(newer, offsets=[1])]
+                )
+            ]
+        ),
+        read_version=dataset.version,
+    )
+
+    val = dataset.to_table().column("val").to_pylist()
+    assert val[1] == 999  # newest overlay wins
+    assert val[4] == 444  # only the older overlay covers offset 4
+
+
+def test_data_overlay_sparse_per_field(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    # Sparse overlay: `id` covers offset {2}, `val` covers offset {3}. The value
+    # file carries one value per field (rank 0 of each field's coverage).
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "sparse.lance",
+        pa.table(
+            {
+                "id": pa.array([777], pa.int32()),
+                "val": pa.array([330], pa.int32()),
+            }
+        ),
+        fields=[0, 1],
+    )
+    assert data_file.fields == [0, 1]
+
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[[2], [3]])
+    op = lance.LanceOperation.DataOverlay(
+        [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+    )
+    dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version)
+
+    result = dataset.to_table()
+    assert result.column("id").to_pylist()[2] == 777
+    assert result.column("val").to_pylist()[3] == 330
+    # Fields resolve independently: id at offset 3 and val at offset 2 fall through.
+    assert result.column("id").to_pylist()[3] == 3
+    assert result.column("val").to_pylist()[2] == 20
+
+
+def test_data_overlay_round_trips_through_fragment_metadata(tmp_path: Path):
+    import json
+
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4])
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+        ),
+        read_version=dataset.version,
+    )
+    overlay_version = dataset.version
+
+    # Reading the fragment surfaces its overlays, stamped with the commit version.
+    metadata = dataset.get_fragments()[0].metadata
+    assert len(metadata.overlays) == 1
+    assert metadata.overlays[0].offsets == [1, 4]
+    assert metadata.overlays[0].committed_version == overlay_version
+
+    # The overlays survive a JSON round-trip of the metadata.
+    restored = lance.fragment.FragmentMetadata.from_json(json.dumps(metadata.to_json()))
+    assert len(restored.overlays) == 1
+    assert restored.overlays[0].offsets == [1, 4]
+    assert restored.overlays[0].committed_version == overlay_version
+
+    # A commit that round-trips the fragment (here an Overwrite) must keep the
+    # overlays, so the overlay still resolves on read instead of being dropped.
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.Overwrite(dataset.schema, [restored]),
+        read_version=dataset.version,
+    )
+    result = dataset.to_table()
+    assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90]
+    assert result.column("id").to_pylist() == list(range(10))
+
+
+def test_data_overlay_rejects_invalid_offsets(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table({"val": pa.array([0, 1, 2], pa.int32())})
+    dataset = lance.write_dataset(table, base_dir)
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([9], pa.int32())}),
+        fields=[0],
+    )
+
+    # offsets is neither a flat list of ints (dense) nor a list of per-field int
+    # lists (sparse), so the coverage shape can't be resolved.
+    with pytest.raises(ValueError, match="offsets must be a list"):
+        lance.LanceDataset.commit(
+            dataset,
+            lance.LanceOperation.DataOverlay(
+                [
+                    lance.LanceOperation.DataOverlayGroup(
+                        0,
+                        [
+                            lance.LanceOperation.DataOverlayFile(
+                                data_file, offsets=[0, [1]]
+                            )
+                        ],
+                    )
+                ]
+            ),
+            read_version=dataset.version,
+        )
+
+
+@pytest.mark.parametrize(
+    "offsets",
+    [
+        [2, 1],  # dense, descending
+        [1, 1],  # dense, duplicate
+        [[2, 1]],  # sparse, descending
+        [[1, 1]],  # sparse, duplicate
+    ],
+)
+def test_data_overlay_rejects_unsorted_offsets(tmp_path: Path, offsets):
+    # Offsets map positionally to value rows in data_file. A RoaringBitmap would
+    # silently reorder/dedup them, so a non-ascending list must be rejected up
+    # front rather than corrupting the row mapping.
+    base_dir = tmp_path / "test"
+    table = pa.table({"val": pa.array([0, 1, 2], pa.int32())})
+    dataset = lance.write_dataset(table, base_dir)
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([9, 9], pa.int32())}),
+        fields=[0],
+    )
+
+    with pytest.raises(ValueError, match="strictly ascending"):
+        lance.LanceDataset.commit(
+            dataset,
+            lance.LanceOperation.DataOverlay(
+                [
+                    lance.LanceOperation.DataOverlayGroup(
+                        0,
+                        [
+                            lance.LanceOperation.DataOverlayFile(
+                                data_file, offsets=offsets
+                            )
+                        ],
+                    )
+                ]
+            ),
+            read_version=dataset.version,
+        )
+
+
 def test_schema_project_drop_column(tmp_path: Path):
     table = pa.Table.from_pydict({"a": range(100, 200), "b": range(300, 400)})
     base_dir = tmp_path / "test"
diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py
index b05888df31f..11276a2213d 100644
--- a/python/python/tests/test_fragment.py
+++ b/python/python/tests/test_fragment.py
@@ -279,7 +279,7 @@ def test_fragment_meta():
         "file_size_bytes=100), DataFile(path='1.lance', fields=[1], column_indices=[], "
         "file_major_version=0, file_minor_version=0, file_size_bytes=None)], "
         "physical_rows=100, deletion_file=None, row_id_meta=None, "
-        "created_at_version_meta=None, last_updated_at_version_meta=None)"
+        "created_at_version_meta=None, last_updated_at_version_meta=None, overlays=[])"
     )
 
 
@@ -865,3 +865,100 @@ def test_fragment_take_with_json_column(tmp_path):
     assert metas[0] == '{"val":1}'
     assert metas[1] == '{"val":4}'
     assert metas[2] == '{"val":7}'
+
+
+def test_fragment_create_with_json_column(tmp_path):
+    """Test that LanceFragment.create works with Arrow JSON extension type.
+
+    Previously the single-fragment create path skipped the Arrow JSON (Utf8) ->
+    Lance JSON (JSONB LargeBinary) conversion that write_dataset/write_fragments
+    perform, so the raw UTF-8 string bytes were written into a column whose schema
+    declared JSONB. Reads then miss-decoded the bytes and returned garbage.
+    """
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "uid": pa.array(["a", "b", "c", "d"], type=pa.utf8()),
+            "payload": pa.array(
+                ['{"x":1}', '{"x":2}', '{"y":3}', '{"y":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    frag = LanceFragment.create(tmp_path, data)
+    operation = LanceOperation.Overwrite(data.schema, [frag])
+    dataset = LanceDataset.commit(tmp_path, operation)
+
+    result = dataset.to_table()
+    assert result.column("uid").to_pylist() == ["a", "b", "c", "d"]
+    payloads = result.column("payload").to_pylist()
+    assert [json.loads(p) for p in payloads] == [
+        {"x": 1},
+        {"x": 2},
+        {"y": 3},
+        {"y": 4},
+    ]
+
+
+def test_fragment_update_columns_with_json_column(tmp_path):
+    """Test that fragment update_columns works with Arrow JSON extension type.
+
+    Previously this would fail with a type mismatch error because the
+    HashJoiner didn't convert Arrow JSON (Utf8) to Lance JSON (LargeBinary).
+    """
+    # Create initial dataset with a JSON extension type column
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()),
+            "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()),
+            "meta": pa.array(
+                ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'],
+                type=json_type,
+            ),
+        }
+    )
+    dataset_uri = tmp_path / "test_update_cols_json"
+    dataset = lance.write_dataset(data, dataset_uri)
+
+    # Prepare update data: update the JSON column for some rows
+    update_data = pa.table(
+        {
+            "_rowid": pa.array([1, 3], type=pa.uint64()),
+            "meta": pa.array(
+                ['{"updated":true,"id":2}', '{"updated":true,"id":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    # This should NOT raise a type mismatch error
+    fragment = dataset.get_fragment(0)
+    updated_fragment, fields_modified = fragment.update_columns(update_data)
+
+    assert len(fields_modified) > 0
+
+    # Commit and verify
+    op = LanceOperation.Update(
+        updated_fragments=[updated_fragment],
+        fields_modified=fields_modified,
+    )
+    updated_dataset = lance.LanceDataset.commit(
+        str(dataset_uri), op, read_version=dataset.version
+    )
+
+    result = updated_dataset.to_table()
+    ids = result.column("id").to_pylist()
+    metas = result.column("meta").to_pylist()
+
+    for i, (id_val, meta_val) in enumerate(zip(ids, metas)):
+        meta = json.loads(meta_val) if isinstance(meta_val, str) else meta_val
+        if id_val == 2 or id_val == 4:
+            assert "updated" in meta_val or meta.get("updated") is True, (
+                f"id={id_val} should be updated, got {meta_val}"
+            )
+        else:
+            assert "x" in meta_val or "x" in str(meta), (
+                f"id={id_val} should have original value, got {meta_val}"
+            )
diff --git a/python/python/tests/test_geo.py b/python/python/tests/test_geo.py
index c011c2de3de..c2dd599e432 100644
--- a/python/python/tests/test_geo.py
+++ b/python/python/tests/test_geo.py
@@ -19,6 +19,19 @@
 )
 
 
+def _query_point_ids(dataset: lance.LanceDataset, wkt: str) -> list[int]:
+    sql = f"""
+          SELECT id, point
+          FROM dataset
+          WHERE St_Intersects(point, ST_GeomFromText('{wkt}'))
+          """
+    return [
+        value
+        for batch in dataset.sql(sql).build().to_batch_records()
+        for value in batch.column("id").to_pylist()
+    ]
+
+
 def test_geo_types(tmp_path: Path):
     uri = str(tmp_path / "test_geo_types.lance")
     # Points
@@ -153,3 +166,275 @@ def query(ds: lance.LanceDataset, has_index=False):
     table_with_index = query(ds, has_index=True)
 
     assert table_with_index == table_without_index
+
+
+def test_rtree_segment_merge_and_commit(tmp_path: Path):
+    num_points = 120
+    points_2d = points(
+        [
+            np.arange(num_points, dtype=np.float64),
+            np.arange(num_points, dtype=np.float64),
+        ]
+    )
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(point("xy")).with_name("point"),
+        ]
+    )
+    table = pa.Table.from_arrays(
+        [np.arange(num_points, dtype=np.int64), points_2d], schema=schema
+    )
+    ds = lance.write_dataset(
+        table,
+        str(tmp_path / "segmented_rtree.lance"),
+        max_rows_per_file=40,
+    )
+    fragments = ds.get_fragments()
+    assert len(fragments) == 3
+    segments = [
+        ds.create_index_uncommitted(
+            column="point",
+            index_type="RTREE",
+            name="point_rtree",
+            fragment_ids=[fragment.fragment_id],
+        )
+        for fragment in fragments
+    ]
+
+    merged = ds.merge_existing_index_segments(segments)
+    assert set(merged.fragment_ids) == {fragment.fragment_id for fragment in fragments}
+    ds = ds.commit_existing_index_segments("point_rtree", "point", [merged])
+
+    sql = """
+          SELECT id, point
+          FROM dataset
+          WHERE St_Intersects(point, ST_GeomFromText('LINESTRING (10 10, 110 110)'))
+          """
+    indexed = pa.Table.from_batches(ds.sql(sql).build().to_batch_records())
+    assert indexed["id"].to_pylist() == list(range(10, 111))
+    explain = (
+        pa.Table.from_batches(
+            ds.sql("EXPLAIN ANALYZE " + sql).build().to_batch_records()
+        )
+        .to_pandas()
+        .to_string()
+    )
+    assert "ScalarIndexQuery" in explain
+
+
+def test_staged_rtree_after_rewrite_columns(tmp_path: Path):
+    uri = str(tmp_path / "stale_rtree.lance")
+    point_type = point("xy")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+    dataset = lance.write_dataset(
+        pa.Table.from_arrays(
+            [
+                pa.array([0, 1], type=pa.int64()),
+                points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]),
+            ],
+            schema=schema,
+        ),
+        uri,
+    )
+    segment = dataset.create_index_uncommitted(
+        column="point",
+        index_type="RTREE",
+        name="point_rtree",
+        fragment_ids=[0],
+    )
+
+    update_schema = pa.schema(
+        [
+            pa.field("_rowid", pa.uint64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+    update = pa.Table.from_arrays(
+        [
+            pa.array([0], type=pa.uint64()),
+            points([np.array([10.0]), np.array([10.0])]),
+        ],
+        schema=update_schema,
+    )
+    fragment, fields = dataset.get_fragment(0).update_columns(update)
+    updated = lance.LanceDataset.commit(
+        uri,
+        lance.LanceOperation.Update(
+            updated_fragments=[fragment],
+            fields_modified=fields,
+        ),
+        read_version=dataset.version,
+    )
+
+    assert _query_point_ids(updated, "POINT (10 10)") == [0]
+    committed = updated.commit_existing_index_segments(
+        "point_rtree",
+        "point",
+        [segment],
+    )
+    assert _query_point_ids(committed, "POINT (10 10)") == [0]
+
+
+def test_rtree_rejects_distributed_uuid_reuse(tmp_path: Path):
+    uri = str(tmp_path / "uuid_reuse.lance")
+    num_points = 120
+    point_type = point("xy")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+    dataset = lance.write_dataset(
+        pa.Table.from_arrays(
+            [
+                pa.array(range(num_points), type=pa.int64()),
+                points(
+                    [
+                        np.arange(num_points, dtype=np.float64),
+                        np.arange(num_points, dtype=np.float64),
+                    ]
+                ),
+            ],
+            schema=schema,
+        ),
+        uri,
+        max_rows_per_file=40,
+    )
+    dataset.create_scalar_index("point", "RTREE")
+    index_uuid = dataset.describe_indices()[0].segments[0].uuid
+
+    with pytest.raises(
+        ValueError,
+        match="index_uuid is no longer accepted for RTree distributed index builds",
+    ):
+        dataset.create_index_uncommitted(
+            column="point",
+            index_type="RTREE",
+            name="point_rtree_reuse",
+            fragment_ids=[0],
+            index_uuid=index_uuid,
+        )
+
+    assert _query_point_ids(
+        lance.dataset(uri),
+        "LINESTRING (100 100, 110 110)",
+    ) == list(range(100, 111))
+
+
+def test_rtree_merge_all_deleted_stable_row_ids(tmp_path: Path):
+    uri = str(tmp_path / "all_deleted.lance")
+    point_type = point("xy")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+    dataset = lance.write_dataset(
+        pa.Table.from_arrays(
+            [
+                pa.array([0, 1], type=pa.int64()),
+                points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]),
+            ],
+            schema=schema,
+        ),
+        uri,
+        enable_stable_row_ids=True,
+    )
+    segment = dataset.create_index_uncommitted(
+        column="point",
+        index_type="RTREE",
+        name="point_rtree",
+        fragment_ids=[0],
+    )
+
+    dataset.delete("true")
+    merged = dataset.merge_existing_index_segments([segment])
+    assert merged.fragment_ids == set()
+    committed = dataset.commit_existing_index_segments(
+        "point_rtree",
+        "point",
+        [merged],
+    )
+    assert _query_point_ids(committed, "POINT (0 0)") == []
+
+
+def test_rtree_merge_preserves_newer_fragment_coverage(tmp_path: Path):
+    uri = str(tmp_path / "mixed_versions.lance")
+    point_type = point("xy")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+
+    def batch(start: int, stop: int) -> pa.Table:
+        values = np.arange(start, stop, dtype=np.float64)
+        return pa.Table.from_arrays(
+            [
+                pa.array(range(start, stop), type=pa.int64()),
+                points([values, values]),
+            ],
+            schema=schema,
+        )
+
+    dataset = lance.write_dataset(batch(0, 40), uri)
+    first = dataset.create_index_uncommitted(
+        column="point",
+        index_type="RTREE",
+        name="point_rtree",
+        fragment_ids=[0],
+    )
+    dataset = lance.write_dataset(batch(40, 80), uri, mode="append")
+    second = dataset.create_index_uncommitted(
+        column="point",
+        index_type="RTREE",
+        name="point_rtree",
+        fragment_ids=[1],
+    )
+
+    merged = dataset.merge_existing_index_segments([first, second])
+    assert merged.fragment_ids == {0, 1}
+    assert merged.dataset_version == dataset.version
+
+    update_schema = pa.schema(
+        [
+            pa.field("_rowid", pa.uint64()),
+            pa.field(point_type).with_name("point"),
+        ]
+    )
+    update = pa.Table.from_arrays(
+        [
+            pa.array([1 << 32], type=pa.uint64()),
+            points([np.array([100.0]), np.array([100.0])]),
+        ],
+        schema=update_schema,
+    )
+    fragment, fields = dataset.get_fragment(1).update_columns(update)
+    dataset = lance.LanceDataset.commit(
+        uri,
+        lance.LanceOperation.Update(
+            updated_fragments=[fragment],
+            fields_modified=fields,
+        ),
+        read_version=dataset.version,
+    )
+
+    committed = dataset.commit_existing_index_segments(
+        "point_rtree",
+        "point",
+        [merged],
+    )
+    assert committed.describe_indices()[0].segments[0].fragment_ids == {0}
+    assert _query_point_ids(
+        committed,
+        "POINT (100 100)",
+    ) == [40]
diff --git a/python/python/tests/test_indices.py b/python/python/tests/test_indices.py
index 02cf64541d6..ae51e6ddb0d 100644
--- a/python/python/tests/test_indices.py
+++ b/python/python/tests/test_indices.py
@@ -8,7 +8,7 @@
 import numpy as np
 import pyarrow as pa
 import pytest
-from lance.file import LanceFileReader
+from lance.file import LanceFileReader, LanceFileWriter
 from lance.indices import IndicesBuilder, IvfModel, PqModel
 
 NUM_ROWS_PER_FRAGMENT = 10000
@@ -209,6 +209,29 @@ def test_gen_pq(tmpdir, rand_dataset, rand_ivf):
     assert pq.dimension == reloaded.dimension
     assert pq.codebook == reloaded.codebook
 
+    pq_4bit = IndicesBuilder(rand_dataset, "vectors").train_pq(
+        rand_ivf,
+        sample_rate=2,
+        num_bits=4,
+    )
+    assert pq_4bit.num_bits == 4
+    assert len(pq_4bit.codebook) == 16
+
+    pq_4bit.save(str(tmpdir / "pq_4bit"))
+    reloaded = PqModel.load(str(tmpdir / "pq_4bit"))
+    assert reloaded.num_bits == 4
+
+    legacy_pq_uri = str(tmpdir / "legacy_pq")
+    with LanceFileWriter(
+        legacy_pq_uri,
+        pa.schema(
+            [pa.field("codebook", pq.codebook.type)],
+            metadata={b"num_subvectors": str(pq.num_subvectors).encode()},
+        ),
+    ) as writer:
+        writer.write_batch(pa.table([pq.codebook], names=["codebook"]))
+    assert PqModel.load(legacy_pq_uri).num_bits == 8
+
 
 def test_ivf_centroids_fragment_ids(tmpdir):
     rows_per_fragment = 32
@@ -300,7 +323,7 @@ def test_indices_builder_multivector_distributed_dimensions(tmpdir, monkeypatch)
 
     captured_dimensions = {}
 
-    def train_pq_model(*args):
+    def train_pq_model(*args, **kwargs):
         captured_dimensions["train_pq"] = args[2]
         return codebook
 
diff --git a/python/python/tests/test_lance.py b/python/python/tests/test_lance.py
index 0162e370665..2ad3af2e029 100644
--- a/python/python/tests/test_lance.py
+++ b/python/python/tests/test_lance.py
@@ -248,6 +248,28 @@ def test_io_counters(tmp_path):
     assert lance.bytes_read_counter() > starting_bytes
 
 
+def test_simd_info():
+    info = lance.simd_info()
+    assert info["tier"] in (
+        "none",
+        "sse",
+        "avx",
+        "avx_fma",
+        "avx2",
+        "avx512",
+        "avx512_fp16",
+        "neon",
+        "lsx",
+        "lasx",
+    )
+    assert isinstance(info["target_arch"], str) and info["target_arch"]
+    if info["target_arch"] == "x86_64":
+        # The x86_64 ABI mandates SSE2.
+        assert "sse2" in info["host_features"]
+    else:
+        assert info["host_features"] == []
+
+
 @pytest.mark.parametrize(
     "row_param, column_name",
     [("with_row_id", "_rowid"), ("with_row_address", "_rowaddr")],
diff --git a/python/python/tests/test_mem_wal.py b/python/python/tests/test_mem_wal.py
index 95596a7123e..4a688c4168b 100644
--- a/python/python/tests/test_mem_wal.py
+++ b/python/python/tests/test_mem_wal.py
@@ -9,6 +9,7 @@
 import pyarrow as pa
 import pytest
 from lance.mem_wal import (
+    CompactedSsTable,
     LsmPointLookupPlanner,
     LsmScanner,
     LsmVectorSearchPlanner,
@@ -56,15 +57,15 @@ def _append_only_table(ids, prefix: str) -> pa.Table:
     )
 
 
-def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa.Table):
-    """Write a flushed-generation Lance dataset at the expected sub-path.
+def _write_sstable(base_path: str, shard_id: str, gen_folder: str, data: pa.Table):
+    """Write an SSTable Lance dataset at the expected sub-path.
 
-    The collector resolves flushed generation paths as:
+    The collector resolves SSTable paths as:
         {base_dataset_path}/_mem_wal/{shard_id}/{gen_folder}
 
     Production flush also writes a primary-key dedup sidecar (`_pk_index/`) that
     the LSM scanner opens to dedup across generations; stage it here too so the
-    flushed generation faithfully matches what flush produces.
+    SSTable faithfully matches what flush produces.
     """
     from lance.lance import _write_pk_sidecar
 
@@ -73,17 +74,36 @@ def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa.
     _write_pk_sidecar(gen_path, data, ["id"])
 
 
+def test_mark_sstables_as_compacted(tmp_path):
+    ds_path = str(tmp_path / "base")
+    shard_id = str(uuid.uuid4())
+    dataset = lance.write_dataset(
+        _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA
+    )
+    dataset.initialize_mem_wal()
+
+    (
+        dataset.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .mark_sstables_as_compacted([CompactedSsTable(shard_id, 1)])
+        .execute(_lookup_table([2, 4], "compacted"))
+    )
+
+    assert dataset.count_rows() == 4
+
+
 def test_point_lookup_with_memtables(tmp_path):
     """
-    Lookup against a base table that has one flushed generation containing an
-    update.  The flushed version must win over the base table version.
+    Lookup against a base table that has one SSTable containing an
+    update.  The SSTable version must win over the base table version.
 
     Setup
     -----
     base   : ids [1, 2, 3]  names ["base_1",  "base_2",  "base_3"]
     gen_1  : ids [2]        names ["gen1_2"]          ← update to id=2
 
-    ShardSnapshot: flushed_generation(gen=1, path="gen_1"), current_generation=2
+    ShardSnapshot: sstable(gen=1, path="gen_1"), current_generation=2
     """
     ds_path = str(tmp_path / "base")
     shard_id = str(uuid.uuid4())
@@ -94,28 +114,25 @@ def test_point_lookup_with_memtables(tmp_path):
     )
     base_ds.initialize_mem_wal()
 
-    # --- Flushed generation: overwrites id=2 ---
-    _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1"))
+    # --- SSTable: overwrites id=2 ---
+    _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1"))
 
-    # --- ShardSnapshot describing the flushed state ---
-    snap = (
-        ShardSnapshot(shard_id)
-        .with_flushed_generation(1, "gen_1")
-        .with_current_generation(2)
-    )
+    # --- ShardSnapshot describing the SSTable state ---
+    snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2)
 
     planner = LsmPointLookupPlanner(base_ds, [snap])
     assert not hasattr(planner, "lookup")
 
-    # id=2 must return the flushed version
+    # id=2 must return the SSTable version
     plan = planner.plan_lookup(pa.array([2], type=pa.int64()))
     assert plan.schema.names == ["id", "name"]
     assert plan.dataset_schema.names == ["id", "name"]
-    assert "Take" in plan.explain() or "Scan" in plan.explain()
+    explained = plan.explain()
+    assert "Take" in explained or "Scan" in explained or "LanceRead" in explained
     result = plan.to_table()
     assert len(result) == 1, "Expected exactly one row for id=2"
     assert result.column("name")[0].as_py() == "gen1_2", (
-        "Flushed generation must win over base table"
+        "SSTable must win over base table"
     )
 
     # id=1 is only in the base table
@@ -146,13 +163,9 @@ def test_lsm_scanner_with_memtables(tmp_path):
     )
     base_ds.initialize_mem_wal()
 
-    _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1"))
+    _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1"))
 
-    snap = (
-        ShardSnapshot(shard_id)
-        .with_flushed_generation(1, "gen_1")
-        .with_current_generation(2)
-    )
+    snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2)
 
     scanner = LsmScanner.from_snapshots(base_ds, [snap])
     table = scanner.to_table()
@@ -161,7 +174,7 @@ def test_lsm_scanner_with_memtables(tmp_path):
     name_by_id = {row["id"]: row["name"] for row in table.to_pylist()}
 
     assert name_by_id[1] == "base_1"
-    assert name_by_id[2] == "gen1_2", "Flushed gen must overwrite base for id=2"
+    assert name_by_id[2] == "gen1_2", "SSTable gen must overwrite base for id=2"
     assert name_by_id[3] == "base_3"
 
     offset_table = (
@@ -170,7 +183,7 @@ def test_lsm_scanner_with_memtables(tmp_path):
     assert len(offset_table) == 2, "Offset-only LSM scan should not require a limit"
 
 
-def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path):
+def test_shard_writer_lsm_scanner_includes_own_sstables(tmp_path):
     ds_path = str(tmp_path / "base")
     shard_id = str(uuid.uuid4())
     ds = lance.write_dataset(_lookup_table([0], "base"), ds_path, schema=_LOOKUP_SCHEMA)
@@ -192,10 +205,37 @@ def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path):
             if name_by_id.get(1) == "writer_1" and name_by_id.get(2) == "writer_2":
                 break
             if time.time() >= deadline:
-                assert False, "writer.lsm_scanner() did not include flushed writer rows"
+                assert False, "writer.lsm_scanner() did not include SSTable writer rows"
             time.sleep(0.05)
 
 
+def test_shard_writer_delete_binding_masks_base_row(tmp_path):
+    ds_path = str(tmp_path / "base")
+    shard_id = str(uuid.uuid4())
+    ds = lance.write_dataset(
+        _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA
+    )
+    ds.initialize_mem_wal()
+
+    delete_keys = pa.table({"id": pa.array([2], type=pa.int64())})
+
+    with ds.mem_wal_writer(
+        shard_id,
+        durable_write=True,
+        max_wal_buffer_size=1,
+        max_wal_flush_interval_ms=10,
+    ) as writer:
+        writer.put(_lookup_table([4], "writer"))
+        writer.delete(delete_keys)
+        table = writer.lsm_scanner().to_table()
+
+    rows = {row["id"]: row["name"] for row in table.to_pylist()}
+    assert rows[1] == "base_1"
+    assert 2 not in rows, "deleted base row should be masked by the tombstone"
+    assert rows[3] == "base_3"
+    assert rows[4] == "writer_4"
+
+
 _VDIM = 4  # matches Rust test fixture dimension
 
 
@@ -305,7 +345,7 @@ def test_shard_writer_e2e_correctness(tmp_path):
     End-to-end correctness test for ShardWriter covering:
     - Multi-round writes that trigger WAL and MemTable flushes
     - File-system layout verification (_mem_wal/<shard_id>/wal/ and manifest/)
-    - Flushed generation data readable via LsmScanner
+    - SSTable data readable via LsmScanner
     - New writer created after close can write and scan correctly
 
     Mirrors Rust test: shard_writer_tests::test_shard_writer_e2e_correctness
@@ -327,7 +367,6 @@ def test_shard_writer_e2e_correctness(tmp_path):
     writer = ds.mem_wal_writer(
         shard_id,
         durable_write=True,
-        sync_indexed_write=True,
         max_wal_buffer_size=10 * 1024,  # 10 KB
         max_wal_flush_interval_ms=50,
         max_memtable_size=80,  # flush after ~80 rows
@@ -377,9 +416,7 @@ def test_shard_writer_e2e_correctness(tmp_path):
     # === New writer: write and read back via active MemTable scanner ===
     ds2 = lance.dataset(ds_path)
     shard_id2 = str(uuid.uuid4())
-    with ds2.mem_wal_writer(
-        shard_id2, durable_write=False, sync_indexed_write=True
-    ) as writer2:
+    with ds2.mem_wal_writer(shard_id2, durable_write=False) as writer2:
         verify_batch = _e2e_batch(schema, start_id=10000, num_rows=10)
         writer2.put(pa.Table.from_batches([verify_batch]))
         result = writer2.lsm_scanner().to_table()
@@ -494,7 +531,6 @@ def test_initialize_mem_wal_writer_config_defaults(tmp_path):
     # Duration knobs are recorded in milliseconds with a `_ms` suffix.
     assert defaults["max_wal_flush_interval_ms"] == "250"
     # Every ShardWriterConfig tunable is recorded once any default is set.
-    assert "sync_indexed_write" in defaults
     assert "enable_memtable" in defaults
 
 
diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py
index e35093dc370..5d88af2566c 100644
--- a/python/python/tests/test_optimize.py
+++ b/python/python/tests/test_optimize.py
@@ -591,6 +591,10 @@ def test_remap_row_addrs(tmp_path: Path):
     before = ds.scanner(columns=["id"], with_row_address=True).to_table()
     old = dict(zip(before["id"].to_pylist(), before["_rowaddr"].to_pylist()))
 
+    # A deferred-remap compaction records a fragment-reuse index only when it
+    # rewrites data an index covers, so index a column first.
+    ds.create_scalar_index("id", "BTREE")
+
     ds.optimize.compact_files(
         target_rows_per_fragment=1_000, defer_index_remap=True, num_threads=1
     )
diff --git a/python/python/tests/test_otel.py b/python/python/tests/test_otel.py
new file mode 100644
index 00000000000..a96e0125f56
--- /dev/null
+++ b/python/python/tests/test_otel.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+import lance
+import pyarrow as pa
+import pytest
+
+# The metrics recorder is process-global and installed once, so the whole
+# bridge is exercised in a single test to avoid cross-test global-state coupling.
+
+
+def _metrics_by_name(reader):
+    data = reader.get_metrics_data()
+    result = {}
+    for resource_metrics in data.resource_metrics:
+        for scope_metrics in resource_metrics.scope_metrics:
+            for metric in scope_metrics.metrics:
+                result[metric.name] = metric
+    return result
+
+
+def test_instrument_lance_metrics_exports_object_store_metrics(tmp_path):
+    pytest.importorskip("opentelemetry.sdk.metrics")
+    from lance.otel import instrument_lance_metrics
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+
+    reader = InMemoryMetricReader()
+    provider = MeterProvider(metric_readers=[reader])
+    assert instrument_lance_metrics(provider)
+
+    # The catalog is populated once the recorder is installed.
+    from lance.lance import lance_metrics_catalog
+
+    catalog = {desc.name: desc for desc in lance_metrics_catalog()}
+    assert "lance_object_store_requests_total" in catalog
+    assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram"
+    # Gauges and the retryable counter are described too, so they surface in the
+    # export even when a plain write doesn't happen to emit them.
+    assert catalog["lance_object_store_retryable_responses_total"].kind == "counter"
+    assert catalog["lance_object_store_in_flight_requests"].kind == "gauge"
+
+    # Generate object store activity on the local filesystem (scheme "file").
+    table = pa.table({"id": pa.array(range(256))})
+    dataset = lance.write_dataset(table, str(tmp_path / "ds.lance"))
+    assert dataset.to_table().num_rows == 256
+
+    metrics = _metrics_by_name(reader)
+
+    requests = metrics["lance_object_store_requests_total"]
+    points = list(requests.data.data_points)
+    assert points, "expected at least one request data point"
+    # The `base` label carries the store scheme ("file") by default.
+    assert all("base" in p.attributes and "operation" in p.attributes for p in points)
+    assert sum(p.value for p in points) > 0
+
+    # Histograms are decomposed into bucket / count / sum observable counters.
+    bucket = metrics["lance_object_store_request_duration_seconds_bucket"]
+    bucket_points = list(bucket.data.data_points)
+    assert bucket_points
+    assert all("le" in p.attributes for p in bucket_points)
+    # The implicit +Inf bucket must be present and is the cumulative maximum.
+    assert any(p.attributes["le"] == "+Inf" for p in bucket_points)
+
+    count = metrics["lance_object_store_request_duration_seconds_count"]
+    assert sum(p.value for p in count.data.data_points) > 0
+
+    # The `_sum` instrument must also be wired and report positive latency.
+    duration_sum = metrics["lance_object_store_request_duration_seconds_sum"]
+    assert sum(p.value for p in duration_sum.data.data_points) > 0
+
+
+def test_snapshot_empty_before_install_is_safe():
+    # snapshot is callable regardless of installation state and never raises.
+    from lance.lance import snapshot_lance_metrics
+
+    assert isinstance(snapshot_lance_metrics(), list)
+
+
+def test_instrument_warns_when_recorder_unavailable(monkeypatch):
+    # A foreign `metrics` recorder already installed -> register returns False;
+    # instrument_lance_metrics must warn and return False without instrumenting.
+    pytest.importorskip("opentelemetry.sdk.metrics")
+    import lance.otel as otel
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+
+    monkeypatch.setattr(otel, "register_lance_metrics_recorder", lambda: False)
+
+    reader = InMemoryMetricReader()
+    provider = MeterProvider(metric_readers=[reader])
+    with pytest.warns(UserWarning, match="recorder"):
+        assert otel.instrument_lance_metrics(provider) is False
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index ae92abdd427..30dab9d5fd2 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -7,6 +7,8 @@
 import re
 import shutil
 import string
+import subprocess
+import sys
 import uuid
 import zipfile
 from datetime import date, datetime, timedelta
@@ -21,6 +23,7 @@
 from lance.query import (
     BooleanQuery,
     BoostQuery,
+    FullTextOperator,
     MatchQuery,
     MultiMatchQuery,
     Occur,
@@ -624,13 +627,13 @@ def make_fts_search(ds):
     plan = make_vec_search(ds).explain_plan()
     assert "ScalarIndexQuery" in plan
     assert "KNNVectorDistance" not in plan
-    assert "LanceRead" not in plan
+    assert "num_fragments" not in plan  # no scan; the take prints as LanceRead
     assert make_vec_search(ds).to_table().num_rows == 6
 
     plan = make_fts_search(ds).explain_plan()
     assert "ScalarIndexQuery" in plan
     assert "KNNVectorDistance" not in plan
-    assert "LanceRead" not in plan
+    assert "num_fragments" not in plan  # no scan; the take prints as LanceRead
     assert make_fts_search(ds).to_table().num_rows == 6
 
     # Add new data (including 6 more results)
@@ -806,6 +809,341 @@ def test_full_text_search(dataset, with_position, base_tokenizer):
         )
 
 
+@pytest.mark.parametrize("block_size", [128, 256])
+def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path, block_size):
+    table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]})
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        block_size=block_size,
+    )
+    assert ds.describe_indices()[0].segments[0].index_version == 3
+
+    results = ds.to_table(
+        columns=["code"],
+        full_text_query=MatchQuery("user", "code"),
+    )
+    assert results["code"].to_pylist() == ["user"]
+
+    stats = ds.stats.index_stats("code_idx")["indices"][0]
+    params = stats["params"]
+    assert "analyzer" not in params
+    assert params["base_tokenizer"] == "code"
+    assert params["split_identifiers"] is False
+
+
+def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path):
+    table = pa.table(
+        {
+            "code": [
+                "getUserName",
+                "set_user_name",
+                "user-name",
+                "username",
+                "other",
+            ]
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        split_identifiers=True,
+    )
+
+    results = ds.to_table(
+        columns=["code"],
+        full_text_query=MatchQuery("user", "code"),
+    )
+    assert set(results["code"].to_pylist()) == {
+        "getUserName",
+        "set_user_name",
+        "user-name",
+    }
+
+    stats = ds.stats.index_stats("code_idx")["indices"][0]
+    params = stats["params"]
+    assert "analyzer" not in params
+    assert params["base_tokenizer"] == "code"
+    assert params["split_identifiers"] is True
+    assert params["split_on_numerics"] is True
+    assert params["preserve_original"] is True
+    assert params["stem"] is False
+    assert params["remove_stop_words"] is False
+
+
+def test_code_analyzer_operator_search_matches_rust_turbofish(tmp_path):
+    table = pa.table(
+        {
+            "path": ["turbofish.rs", "comparison.rs"],
+            "code": ["value.parse::<usize>()", "value.parse<usize>()"],
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        index_operators=True,
+    )
+
+    results = ds.to_table(
+        columns=["path"],
+        full_text_query=MatchQuery("::", "code", operator=FullTextOperator.OR),
+    )
+    assert results["path"].to_pylist() == ["turbofish.rs"]
+
+
+def test_code_analyzer_exact_identifier_survives_grouped_top_k(tmp_path):
+    table = pa.table(
+        {
+            "path": ["split_0.rs", "split_1.rs", "split_2.rs", "exact.rs"],
+            "code": ["get user name", "get user name", "get user name", "getUserName"],
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        split_identifiers=True,
+    )
+
+    results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery(
+            "getUserName", "code", operator=FullTextOperator.AND
+        ),
+        limit=1,
+    ).to_table()
+    assert results["path"].to_pylist() == ["exact.rs"]
+
+
+def test_code_analyzer_flags_require_code_analyzer(tmp_path):
+    table = pa.table({"text": ["getUserName"]})
+    ds = lance.write_dataset(table, tmp_path)
+
+    with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"):
+        ds.create_scalar_index(
+            "text",
+            index_type="INVERTED",
+            split_identifiers=True,
+        )
+
+
+def test_code_analyzer_requires_fts_v3(tmp_path):
+    table = pa.table({"code": ["getUserName"]})
+    ds = lance.write_dataset(table, tmp_path)
+
+    with pytest.raises(ValueError, match="requires FTS format_version=3"):
+        ds.create_scalar_index(
+            "code",
+            index_type="INVERTED",
+            analyzer="code",
+            format_version=2,
+        )
+
+
+def test_code_analyzer_complex_code_constructs(tmp_path):
+    table = pa.table(
+        {
+            "path": [
+                "edge/trait.rs",
+                "edge/impl.rs",
+                "edge/fn_pointer.rs",
+                "edge/unit_result.rs",
+                "edge/hrtb.rs",
+                "edge/associated.rs",
+                "edge/operators.rs",
+            ],
+            "code": [
+                """
+pub trait EdgeAsyncRepository<'a, T: Send + Sync>
+where
+    T: TryFrom<&'a str, Error = EdgeParseError>,
+{
+    type Output<'b>: Iterator<Item = Result<T, EdgeRepoError>>
+    where
+        Self: 'b;
+
+    async fn fetch_by_key<const N: usize>(
+        &'a self,
+        key: [u8; N],
+    ) -> Result<Self::Output<'a>, EdgeRepoError>;
+}
+""",
+                """
+impl<'a, T, S> EdgeAsyncRepository<'a, T> for EdgeStore<S>
+where
+    T: TryFrom<&'a str, Error = EdgeParseError> + Clone + Send + Sync,
+    S: EdgeBackend<T> + ?Sized,
+{
+    type Output<'b> = std::vec::IntoIter<Result<T, EdgeRepoError>> where Self: 'b;
+
+    async fn fetch_by_key<const N: usize>(
+        &'a self,
+        key: [u8; N],
+    ) -> Result<Self::Output<'a>, EdgeRepoError> {
+        self.backend.fetch::<T, N>(key).await
+    }
+}
+""",
+                """
+pub fn build_edge_handler<T, F>(
+    factory: F,
+) -> impl Fn() -> Result<EdgeHandler<T>, EdgeError>
+where
+    F: FnOnce() -> Result<T, EdgeError> + Send + 'static,
+    T: Default + Send + Sync + 'static,
+{
+    move || factory().map(EdgeHandler::new)
+}
+""",
+                """
+pub fn edge_unit_result_callback() -> Result<()> {
+    Ok(())
+}
+""",
+                """
+pub fn edge_higher_ranked<'a, T>(
+    visitor: impl for<'b> Fn(&'b T) -> Result<&'b str, EdgeVisitError>,
+    value: &'a T,
+) -> Result<&'a str, EdgeVisitError> {
+    visitor(value)
+}
+""",
+                """
+pub fn edge_collect_stream<I, E>(items: I) -> Result<Vec<E::Item>, E::Error>
+where
+    I: IntoIterator<Item = E>,
+    E: EdgeExtract,
+{
+    items.into_iter().map(E::extract).collect()
+}
+""",
+                """
+pub fn edge_operator_arrow() -> Result<EdgeArrow, EdgeError> {
+    let variant = EdgeModule::EdgeVariant;
+    if variant != EdgeModule::Default && EdgeMask::enabled() {
+        return Ok(EdgeArrow::new(variant));
+    }
+    Err(EdgeError::empty())
+}
+""",
+            ],
+        }
+    )
+    table = table.append_column("code_ops", table["code"])
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+    ds.create_scalar_index(
+        "code_ops",
+        index_type="INVERTED",
+        analyzer="code",
+        index_operators=True,
+    )
+
+    ds.insert(
+        pa.table(
+            {
+                "path": ["edge/flat_unindexed.rs"],
+                "code": [
+                    """
+pub async fn edge_flat_generic_return<T, E>() -> Result<T, EdgeFlatError>
+where
+    T: TryFrom<String, Error = E> + Send,
+    E: Into<EdgeFlatError>,
+{
+    T::try_from(String::new()).map_err(Into::into)
+}
+"""
+                ],
+                "code_ops": [
+                    """
+pub async fn edge_flat_operator() -> Result<EdgeFlat, EdgeFlatError> {
+    EdgeFlat::try_new() -> Result<EdgeFlat, EdgeFlatError>
+}
+"""
+                ],
+            }
+        )
+    )
+    ds = lance.dataset(tmp_path)
+
+    def assert_search(column, query, expected_path, operator=FullTextOperator.AND):
+        result = ds.scanner(
+            columns=["path", "_score"],
+            full_text_query=MatchQuery(query, column, operator=operator),
+            limit=50,
+        ).to_table()
+        assert expected_path in result["path"].to_pylist()
+
+    assert_search(
+        "code",
+        "EdgeAsyncRepository fetch_by_key TryFrom EdgeRepoError",
+        "edge/trait.rs",
+    )
+    assert_search(
+        "code",
+        "EdgeStore fetch_by_key const usize where Result",
+        "edge/impl.rs",
+    )
+    assert_search(
+        "code",
+        "build_edge_handler FnOnce Result EdgeHandler",
+        "edge/fn_pointer.rs",
+    )
+    assert_search(
+        "code",
+        "edge_unit_result_callback fn () -> Result",
+        "edge/unit_result.rs",
+    )
+    assert_search(
+        "code",
+        "edge_higher_ranked for Fn EdgeVisitError Result",
+        "edge/hrtb.rs",
+    )
+    assert_search(
+        "code",
+        "edge_collect_stream IntoIterator Item Error Result",
+        "edge/associated.rs",
+    )
+    assert_search(
+        "code",
+        "edge_flat_generic_return TryFrom EdgeFlatError Result",
+        "edge/flat_unindexed.rs",
+    )
+    assert_search(
+        "code_ops",
+        "edge_operator_arrow -> Result",
+        "edge/operators.rs",
+    )
+    assert_search(
+        "code_ops",
+        "EdgeModule :: EdgeVariant !=",
+        "edge/operators.rs",
+    )
+    assert_search(
+        "code_ops",
+        "edge_flat_operator -> Result EdgeFlatError",
+        "edge/flat_unindexed.rs",
+    )
+
+    default_operator_results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery("->", "code", operator=FullTextOperator.OR),
+    ).to_table()
+    operator_results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery("->", "code_ops", operator=FullTextOperator.OR),
+    ).to_table()
+    assert default_operator_results.num_rows == 0
+    assert operator_results.num_rows > 0
+
+
 def test_unindexed_full_text_search_on_empty_index(tmp_path):
     # Create fts index on empty table.
     schema = pa.schema({"text": pa.string()})
@@ -949,6 +1287,39 @@ def test_create_scalar_index_fts_alias(dataset):
     assert any(idx.index_type == "Inverted" for idx in dataset.describe_indices())
 
 
+def test_create_scalar_index_fts_block_size(dataset):
+    dataset.create_scalar_index(
+        "doc", index_type="INVERTED", with_position=False, block_size=256
+    )
+    indices = dataset.describe_indices()
+    doc_index = next(index for index in indices if index.name == "doc_idx")
+    assert doc_index.segments[0].index_version == 3
+
+    row = dataset.take(indices=[0], columns=["doc"])
+    query = row.column(0)[0].as_py().split(" ")[0]
+    results = dataset.scanner(columns=["doc"], full_text_query=query).to_table()
+    assert results.num_rows > 0
+
+    with pytest.raises(ValueError, match="block_size"):
+        dataset.create_scalar_index(
+            "doc", index_type="INVERTED", name="doc_invalid_129", block_size=129
+        )
+
+    with pytest.raises(ValueError, match="block_size"):
+        dataset.create_scalar_index(
+            "doc", index_type="INVERTED", name="doc_invalid_512", block_size=512
+        )
+
+    with pytest.raises(ValueError, match="block_size=256"):
+        dataset.create_scalar_index(
+            "doc",
+            index_type="INVERTED",
+            name="doc_invalid_v2_256",
+            block_size=256,
+            format_version=2,
+        )
+
+
 def test_multi_index_create(tmp_path):
     dataset = lance.write_dataset(
         pa.table({"ints": range(1024)}), tmp_path, max_rows_per_file=100
@@ -2165,6 +2536,87 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     assert small_bytes_read < large_bytes_read
 
 
+@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"])
+def test_address_domain_index_with_stable_row_ids(tmp_path: Path, index_type):
+    """Regression test for issue #7434.
+
+    Address-domain scalar indices (zonemap, bloom filter) report matches as
+    physical row addresses. On a stable-row-id dataset a row's stable id differs
+    from its physical address in every fragment except fragment 0, so the index
+    result must be translated back to the row-id domain before it prefilters the
+    scan. Without that translation the index silently drops matching rows in
+    fragments other than fragment 0 (often returning an empty result).
+    """
+
+    # A single value "a" is placed in non-adjacent fragments (0, 2, 4) so its
+    # matching rows span multiple fragments and diverge from fragment 0.
+    def block(v, n):
+        return [v] * n
+
+    vals = (
+        block("a", 5_000)
+        + block("b", 5_000)
+        + block("a", 5_000)
+        + block("c", 5_000)
+        + block("a", 5_000)
+    )
+    tbl = pa.table({"category": pa.array(vals, pa.string()), "id": range(len(vals))})
+    ds = lance.write_dataset(
+        tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True
+    )
+    assert ds.has_stable_row_ids
+    assert len(ds.get_fragments()) == 5
+
+    true_count = ds.scanner(
+        filter="category = 'a'", use_scalar_index=False
+    ).count_rows()
+    assert true_count == 15_000
+
+    ds.create_scalar_index("category", index_type=index_type, replace=True)
+
+    # The index must be consulted and must return the same rows as a full scan.
+    assert "ScalarIndexQuery" in ds.scanner(filter="category = 'a'").explain_plan()
+    indexed = ds.to_table(filter="category = 'a'", columns=["id"])
+    assert indexed.num_rows == true_count
+    assert sorted(indexed["id"].to_pylist()) == sorted(
+        ds.to_table(filter="category = 'a'", columns=["id"], use_scalar_index=False)[
+            "id"
+        ].to_pylist()
+    )
+
+
+def test_zonemap_with_stable_row_ids_after_compaction(tmp_path: Path):
+    """Zonemap results stay correct after a compaction relocates rows under
+    stable row ids (physical address != stable id for the surviving rows)."""
+    ds = lance.write_dataset(
+        pa.table({"x": range(0, 5_000)}),
+        tmp_path,
+        max_rows_per_file=5_000,
+        enable_stable_row_ids=True,
+    )
+    ds = lance.write_dataset(
+        pa.table({"x": range(5_000, 10_000)}),
+        tmp_path,
+        mode="append",
+        max_rows_per_file=5_000,
+        enable_stable_row_ids=True,
+    )
+    # Delete part of the first fragment, then compact so surviving rows keep
+    # their small stable ids but move to a freshly numbered fragment.
+    ds.delete("x >= 1000 AND x < 2000")
+    ds.optimize.compact_files(target_rows_per_fragment=100_000)
+    ds = lance.dataset(tmp_path)
+    assert len(ds.get_fragments()) == 1
+
+    ds.create_scalar_index("x", index_type="ZONEMAP")
+
+    filter_expr = "x >= 6000 AND x <= 6500"
+    assert "ScalarIndexQuery" in ds.scanner(filter=filter_expr).explain_plan()
+    expected = ds.to_table(filter=filter_expr, use_scalar_index=False)["x"].to_pylist()
+    actual = ds.to_table(filter=filter_expr)["x"].to_pylist()
+    assert sorted(actual) == sorted(expected) == list(range(6000, 6501))
+
+
 def test_zonemap_deletion_handling(tmp_path: Path):
     """Test zonemap deletion handling"""
     data = pa.table(
@@ -2194,6 +2646,33 @@ def test_zonemap_deletion_handling(tmp_path: Path):
     assert ids == [0, 2, 4, 6, 8]
 
 
+@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"])
+def test_address_domain_index_not_query_with_stable_row_ids(tmp_path: Path, index_type):
+    """Regression test: != queries return correct results on stable-row-id datasets.
+
+    Address-domain indices (zonemap, bloom filter) search returns physical row
+    addresses. Translation to row IDs happens at the Query leaf before the NOT
+    node is evaluated, so the NOT operates on a correctly translated AllowList.
+    Without the address-to-row-id translation the AllowList contains wrong IDs,
+    and the subsequent NOT excludes the wrong rows.
+    """
+    vals = list(range(5_000)) + list(range(5_000, 10_000))
+    tbl = pa.table({"x": vals})
+    ds = lance.write_dataset(
+        tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True
+    )
+    assert ds.has_stable_row_ids
+
+    ds.create_scalar_index("x", index_type=index_type, replace=True)
+
+    # Without address translation the NOT excludes the wrong rows, producing an
+    # incorrect result set rather than crashing.
+    expected = ds.to_table(filter="x != 42", use_scalar_index=False)["x"].to_pylist()
+    actual = ds.to_table(filter="x != 42")["x"].to_pylist()
+    assert sorted(actual) == sorted(expected)
+    assert len(actual) == 9_999
+
+
 def test_zonemap_index_remapping(tmp_path: Path):
     """Test zonemap index remapping after compaction and optimization"""
     # Create a dataset with 5 fragments by writing data in chunks
@@ -2332,6 +2811,36 @@ def test_json_index():
     )
 
 
+def test_json_index_non_exact_floats():
+    # A JSON-path btree index is trained on the value extracted from the JSON
+    # column, not on the raw column itself, so the index build must sort by
+    # the extracted value rather than assuming the raw column's order matches
+    # it. Without that, range/equality queries silently miss rows whenever
+    # the extracted floats are not exactly representable in float64 (#7485).
+    vals = ['{"latitude": 10.5}', '{"latitude": 40.1}', '{"latitude": -3.2}']
+    tbl = pa.table({"data": pa.array(vals, pa.json_())})
+    ds = lance.write_dataset(tbl, "memory://test")
+    ds.create_scalar_index(
+        "data",
+        IndexConfig(
+            index_type="json",
+            parameters={"target_index_type": "btree", "path": "latitude"},
+        ),
+    )
+
+    for filter in [
+        "json_get_float(data, 'latitude') > 0",
+        "json_get_float(data, 'latitude') >= 10.5",
+        "json_get_float(data, 'latitude') = 40.1",
+        "json_get_float(data, 'latitude') = 10.5",
+        "json_get_float(data, 'latitude') < 100",
+    ]:
+        assert "ScalarIndexQuery" in ds.scanner(filter=filter).explain_plan()
+        assert ds.to_table(filter=filter) == ds.to_table(
+            filter=filter, use_scalar_index=False
+        ), filter
+
+
 def test_null_handling():
     tbl = pa.table(
         {
@@ -2793,6 +3302,15 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     cache_entries_after_query = ds._ds.index_cache_entry_count()
     assert cache_entries_after_query == cache_entries_after_prewarm
 
+    segment_uuid = ds.describe_indices()[0].segments[0].uuid
+    ds = lance.dataset(phrase_path)
+    ds.prewarm_index("fts_idx", with_position=True, index_segments=[segment_uuid])
+    cache_entries_after_prewarm = ds._ds.index_cache_entry_count()
+    results = ds.to_table(full_text_query=PhraseQuery("word word", "fts"))
+    assert results.num_rows == test_table_size
+    cache_entries_after_query = ds._ds.index_cache_entry_count()
+    assert cache_entries_after_query == cache_entries_after_prewarm
+
     with pytest.raises(
         TypeError,
         match="takes 2 positional arguments",
@@ -4224,6 +4742,54 @@ def test_zonemap_segment_merge_and_commit_from_python(tmp_path):
     )
 
 
+def test_bloomfilter_segment_merge_and_commit_from_python(tmp_path):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=3, rows_per_fragment=100
+    )
+
+    index_name = "id_bloomfilter_segments"
+    fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()]
+    staged_segments = [
+        ds.create_index_uncommitted(
+            column="id",
+            index_type="BLOOMFILTER",
+            name=index_name,
+            fragment_ids=[fragment_id],
+        )
+        for fragment_id in fragment_ids
+    ]
+
+    for segment, fragment_id in zip(staged_segments, fragment_ids):
+        assert segment.fragment_ids == {fragment_id}
+        assert any(file.path == "bloomfilter.lance" for file in segment.files)
+
+    merged_segment = ds.merge_existing_index_segments(staged_segments)
+    assert merged_segment.fragment_ids == set(fragment_ids)
+    assert any(file.path == "bloomfilter.lance" for file in merged_segment.files)
+
+    ds = ds.commit_existing_index_segments(index_name, "id", [merged_segment])
+    descriptions = {index.name: index for index in ds.describe_indices()}
+    assert descriptions[index_name].index_type == "BloomFilter"
+    assert len(descriptions[index_name].segments) == 1
+
+    filter_expr = "id = 117"
+    without_index = ds.scanner(
+        filter=filter_expr,
+        columns=["id", "text"],
+        use_scalar_index=False,
+    ).to_table()
+    with_index = ds.scanner(
+        filter=filter_expr,
+        columns=["id", "text"],
+        use_scalar_index=True,
+    ).to_table()
+    assert with_index.to_pydict() == without_index.to_pydict()
+    assert (
+        "ScalarIndexQuery"
+        in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan()
+    )
+
+
 def test_merge_index_metadata_btree_soft_break(tmp_path):
     ds = generate_multi_fragment_dataset(
         tmp_path, num_fragments=2, rows_per_fragment=100
@@ -4719,6 +5285,77 @@ def test_nested_field_fts_index(tmp_path):
     assert results.num_rows == 50
 
 
+def test_multiple_nested_field_fts_indices_e2e(tmp_path):
+    """Test FTS queries against multiple indexed nested string fields."""
+
+    def make_table(ids, text_values, summary_values):
+        return pa.table(
+            {
+                "id": ids,
+                "data": pa.StructArray.from_arrays(
+                    [
+                        pa.array(text_values, type=pa.string()),
+                        pa.array(summary_values, type=pa.string()),
+                    ],
+                    names=["text", "summary"],
+                ),
+            }
+        )
+
+    def result_ids(query):
+        return sorted(ds.to_table(full_text_query=query)["id"].to_pylist())
+
+    ds = lance.write_dataset(
+        make_table(
+            [0, 1, 2, 3],
+            [
+                "lance nested alpha",
+                "plain text",
+                None,
+                "phrase target here",
+            ],
+            [
+                "metadata only",
+                "database nested beta",
+                "lance beta",
+                "other",
+            ],
+        ),
+        tmp_path,
+    )
+
+    ds.create_scalar_index("data.text", index_type="INVERTED", with_position=True)
+    ds.create_scalar_index("data.summary", index_type="INVERTED", with_position=False)
+
+    indexed_fields = {
+        tuple(index.field_names)
+        for index in ds.describe_indices()
+        if index.index_type == "Inverted"
+    }
+    assert indexed_fields == {("data.text",), ("data.summary",)}
+
+    assert result_ids(MatchQuery("alpha", "data.text")) == [0]
+    assert result_ids(MatchQuery("beta", "data.summary")) == [1, 2]
+    assert result_ids("lance") == [0, 2]
+    assert result_ids(MultiMatchQuery("nested", ["data.text", "data.summary"])) == [
+        0,
+        1,
+    ]
+    assert result_ids(PhraseQuery("phrase target", "data.text")) == [3]
+
+    ds = lance.write_dataset(
+        make_table(
+            [4, 5],
+            ["fresh lance append", "plain append"],
+            ["other", "fresh beta append"],
+        ),
+        tmp_path,
+        mode="append",
+    )
+
+    assert result_ids("fresh") == [4, 5]
+
+
 def test_nested_field_bitmap_index(tmp_path):
     """Test BITMAP index creation and querying on nested fields"""
     # Create dataset with nested categorical field
@@ -4853,7 +5490,7 @@ def test_json_inverted_match_query(tmp_path):
 
 @pytest.mark.parametrize(
     ("format_version", "expected_format_version"),
-    [(1, 1), (2, 2), ("v1", 1), ("v2", 2)],
+    [(1, 1), (2, 2), (3, 3), ("v1", 1), ("v2", 2), ("v3", 3)],
 )
 def test_describe_indices(tmp_path, format_version, expected_format_version):
     data = pa.table(
@@ -4903,7 +5540,6 @@ def test_describe_indices(tmp_path, format_version, expected_format_version):
     assert details["lower_case"]
     assert details["stem"]
     assert details["remove_stop_words"]
-    assert details["custom_stop_words"] is None
     assert details["ascii_folding"]
     assert details["min_ngram_length"] == 3
     assert details["max_ngram_length"] == 3
@@ -4984,15 +5620,92 @@ def test_describe_indices(tmp_path, format_version, expected_format_version):
         assert index.num_rows_indexed == 50
 
 
-def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypatch):
-    monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", "1")
-    data = pa.table({"text": ["document about lance database"]})
-    ds = lance.write_dataset(data, tmp_path)
+def _run_fts_format_creation_probe(
+    tmp_path, env_value, creation_options=None, expected_format_version=None
+):
+    script = """
+import json
+import sys
 
-    ds.create_scalar_index("text", index_type="INVERTED")
+import lance
+import pyarrow as pa
 
-    indices = ds.describe_indices()
-    assert indices[0].segments[0].index_version == 2
+dataset = lance.write_dataset(
+    pa.table({"text": ["document about lance database"]}), sys.argv[1]
+)
+dataset.create_scalar_index(
+    "text", index_type="INVERTED", **json.loads(sys.argv[2])
+)
+expected_format_version = json.loads(sys.argv[3])
+if expected_format_version is not None:
+    actual_format_version = dataset.describe_indices()[0].segments[0].index_version
+    assert actual_format_version == expected_format_version
+"""
+    env = os.environ.copy()
+    if env_value is None:
+        env.pop("LANCE_FTS_FORMAT_VERSION", None)
+    else:
+        env["LANCE_FTS_FORMAT_VERSION"] = env_value
+    return subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            script,
+            str(tmp_path),
+            json.dumps(creation_options or {}),
+            json.dumps(expected_format_version),
+        ],
+        capture_output=True,
+        env=env,
+        text=True,
+    )
+
+
+@pytest.mark.parametrize(
+    ("env_value", "creation_options", "expected_format_version"),
+    [
+        ("1", {}, 1),
+        ("2", {}, 2),
+        ("3", {}, 3),
+        ("3", {"block_size": 256}, 3),
+    ],
+)
+def test_create_inverted_index_uses_env_format_version(
+    tmp_path, env_value, creation_options, expected_format_version
+):
+    result = _run_fts_format_creation_probe(
+        tmp_path,
+        env_value,
+        creation_options,
+        expected_format_version,
+    )
+
+    assert result.returncode == 0, result.stderr
+
+
+def test_create_inverted_index_explicit_format_version_overrides_env(tmp_path):
+    result = _run_fts_format_creation_probe(
+        tmp_path,
+        "invalid",
+        {"format_version": 1},
+        1,
+    )
+
+    assert result.returncode == 0, result.stderr
+
+
+def test_create_text_inverted_index_defaults_to_v2_without_env(tmp_path):
+    result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=2)
+
+    assert result.returncode == 0, result.stderr
+
+
+def test_create_inverted_index_rejects_invalid_env_format_version(tmp_path):
+    result = _run_fts_format_creation_probe(tmp_path, "invalid")
+
+    assert result.returncode != 0
+    assert "LANCE_FTS_FORMAT_VERSION" in result.stderr
+    assert "invalid" in result.stderr
 
 
 def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
@@ -5000,7 +5713,10 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
     ds = lance.write_dataset(data, tmp_path)
 
     with pytest.raises(ValueError, match="unsupported FTS format version"):
-        ds.create_scalar_index("text", index_type="INVERTED", format_version="v3")
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v5")
+
+    with pytest.raises(ValueError, match="unsupported FTS format version"):
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v4")
 
 
 def test_vector_filter_fts_search(tmp_path):
diff --git a/python/python/tests/test_schema.py b/python/python/tests/test_schema.py
index fcff283ebe2..c384466082f 100644
--- a/python/python/tests/test_schema.py
+++ b/python/python/tests/test_schema.py
@@ -6,6 +6,7 @@
 
 import lance
 import pyarrow as pa
+import pytest
 from lance.schema import LanceSchema
 
 
@@ -60,3 +61,15 @@ def test_lance_schema(tmp_path: Path):
     s_fields = fields[1].children()
     assert s_fields[0].name() == "new_name"
     assert s_fields[0].id() == 2
+
+
+def test_lance_schema_from_protos_rejects_missing_parent():
+    # name (field 2): child; id (field 3): 7; parent_id (field 4): 42;
+    # logical_type (field 5): int32.
+    field_proto = b"\x12\x05child\x18\x07\x20\x2a\x2a\x05int32"
+
+    with pytest.raises(
+        ValueError,
+        match="Field 'child' \\(id=7\\) references parent id 42",
+    ):
+        LanceSchema._from_protos("{}", field_proto)
diff --git a/python/python/tests/test_tf.py b/python/python/tests/test_tf.py
deleted file mode 100644
index 3652df0e938..00000000000
--- a/python/python/tests/test_tf.py
+++ /dev/null
@@ -1,351 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-import os
-import warnings
-
-import lance
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-import pytest
-from lance.arrow import ImageArray
-from lance.fragment import LanceFragment
-
-pytest.skip("Skip tensorflow tests", allow_module_level=True)
-
-try:
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", category=DeprecationWarning)
-        import tensorflow as tf  # noqa: F401
-except ImportError:
-    pytest.skip(
-        "Tensorflow is not installed. Please install tensorflow to "
-        + "test lance.tf module.",
-        allow_module_level=True,
-    )
-
-from lance.tf.data import (  # noqa: E402
-    from_lance,
-    from_lance_batches,
-    lance_fragments,
-    lance_take_batches,
-)
-
-
-@pytest.fixture
-def tf_dataset(tmp_path):
-    df = pd.DataFrame(
-        {
-            "a": range(10000),
-            "s": [f"val-{i}" for i in range(10000)],
-            "vec": [[i * 0.2] * 128 for i in range(10000)],
-        }
-    )
-
-    schema = pa.schema(
-        [
-            pa.field("a", pa.int64()),
-            pa.field("s", pa.string()),
-            pa.field("vec", pa.list_(pa.float32(), 128)),
-        ]
-    )
-    tbl = pa.Table.from_pandas(df, schema=schema)
-    uri = tmp_path / "dataset.lance"
-    lance.write_dataset(
-        tbl,
-        uri,
-        schema=tbl.schema,
-        max_rows_per_group=100,
-        max_rows_per_file=1000,
-    )
-    return uri
-
-
-def test_fragment_dataset(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8")
-        assert batch["a"].shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_projection(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100, columns=["a"])
-
-    for idx, batch in enumerate(ds):
-        assert list(batch.keys()) == ["a"]
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["a"].shape == (100,)
-
-
-def test_filter(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100, filter="a >= 5000")
-
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100 + 5000
-        assert batch["a"].shape == (100,)
-
-
-def test_namespace_table_id(monkeypatch):
-    calls = {}
-
-    class DummyScanner:
-        def __init__(self):
-            self._batch = pa.record_batch([pa.array([1, 2])], names=["a"])
-            self.projected_schema = self._batch.schema
-
-        def to_batches(self):
-            yield self._batch
-
-    class DummyDataset:
-        def scanner(self, **kwargs):
-            return DummyScanner()
-
-    def fake_dataset(uri=None, **kwargs):
-        calls["uri"] = uri
-        calls["kwargs"] = kwargs
-        return DummyDataset()
-
-    monkeypatch.setattr(lance, "dataset", fake_dataset)
-
-    ns = object()
-    ds = from_lance(
-        None,
-        namespace_client=ns,
-        table_id=["tbl"],
-        ignore_namespace_table_storage_options=True,
-    )
-
-    assert calls["kwargs"]["namespace_client"] is ns
-    assert calls["kwargs"]["table_id"] == ["tbl"]
-    assert calls["kwargs"]["ignore_namespace_table_storage_options"] is True
-
-    batches = list(ds)
-    assert [b["a"].numpy().tolist() for b in batches] == [[1, 2]]
-
-
-def test_scan_use_tf_data(tf_dataset):
-    ds = tf.data.Dataset.from_lance(tf_dataset)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8")
-        assert batch["a"].shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_pass_fragments(tf_dataset):
-    # Can pass fragments directly to from_lance
-    dataset = lance.dataset(tf_dataset)
-    ds = from_lance(tf_dataset, fragments=dataset.get_fragments(), batch_size=100)
-    ds_default = from_lance(tf_dataset, batch_size=100)
-    for batch, batch_default in zip(ds, ds_default):
-        assert batch["a"].numpy()[0] == batch_default["a"].numpy()[0]
-        assert batch["a"].numpy().shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )
-
-    # Can pass ids directly to from_lance
-    ds = from_lance(tf_dataset, fragments=[0, 1, 2], batch_size=100)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["a"].numpy().shape == (100,)
-
-
-def test_shuffle(tf_dataset):
-    fragments = lance_fragments(tf_dataset).shuffle(4, seed=20).take(3)
-
-    ds = from_lance(tf_dataset, fragments=fragments, batch_size=100)
-    raw_ds = lance.dataset(tf_dataset)
-    scanner = raw_ds.scanner(
-        fragments=[LanceFragment(raw_ds, fid) for fid in [0, 3, 1]], batch_size=100
-    )
-
-    for batch, raw_batch in zip(ds, scanner.to_batches()):
-        assert batch["a"].numpy()[0] == raw_batch.to_pydict()["a"][0]
-        assert batch["a"].numpy().shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_dataset_batches(tf_dataset):
-    tf_dataset = lance.dataset(tf_dataset)
-    batch_size = 300
-    batches = list(
-        from_lance_batches(tf_dataset, batch_size=batch_size).as_numpy_iterator()
-    )
-    assert tf_dataset.count_rows() // batch_size + 1 == len(batches)
-    assert all(end - start == batch_size for start, end in batches[:-2])
-    assert batches[-1][1] - batches[-1][0] == tf_dataset.count_rows() % batch_size
-
-    skip = 5
-    batches_skipped = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, skip=skip
-        ).as_numpy_iterator()
-    )
-    assert batches_skipped == batches[skip:]
-
-    batches_shuffled = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, shuffle=True, seed=42
-        ).as_numpy_iterator()
-    )
-    # make sure it does a shuffle
-    assert batches_shuffled != batches
-    batches_shuffled2 = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, shuffle=True, seed=42
-        ).as_numpy_iterator()
-    )
-    # make sure the shuffle can be deterministic
-    assert batches_shuffled == batches_shuffled2
-
-
-def test_take_dataset(tf_dataset):
-    tf_dataset = lance.dataset(tf_dataset)
-    batch_ds = from_lance_batches(
-        tf_dataset, batch_size=100, shuffle=True, seed=42
-    ).as_numpy_iterator()
-    lance_ds = lance_take_batches(tf_dataset, batch_ds)
-    lance_ds = lance_ds.unbatch().shuffle(400, seed=42).batch(100)
-
-    for batch in lance_ds:
-        assert batch["a"].numpy().shape == (100,)
-
-    batches = [(0, 200), (100, 200)]
-    lance_ds = lance_take_batches(tf_dataset, batches, columns=["a"])
-    for (start, end), batch in zip(batches, lance_ds):
-        assert batch["a"].numpy().tolist() == np.arange(start, end).tolist()
-        assert batch.keys() == {"a"}
-
-
-def test_var_length_list(tmp_path):
-    """Treat var length list as RaggedTensor."""
-    df = pd.DataFrame(
-        {
-            "a": range(200),
-            "l": [[i] * (i % 5 + 1) for i in range(200)],
-        }
-    )
-
-    schema = pa.schema(
-        [
-            pa.field("a", pa.int64()),
-            pa.field("l", pa.list_(pa.int32())),
-        ]
-    )
-    tbl = pa.Table.from_pandas(df, schema=schema)
-
-    uri = tmp_path / "dataset.lance"
-    lance.write_dataset(
-        tbl,
-        uri,
-        schema=tbl.schema,
-    )
-
-    output_signature = {
-        "a": tf.TensorSpec(shape=(None,), dtype=tf.int64),
-        "l": tf.RaggedTensorSpec(dtype=tf.dtypes.int32, shape=(8, None), ragged_rank=1),
-    }
-
-    ds = tf.data.Dataset.from_lance(
-        uri,
-        batch_size=8,
-        output_signature=output_signature,
-    )
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 8
-        assert batch["l"].shape == (8, None)
-        assert isinstance(batch["l"], tf.RaggedTensor)
-
-
-def test_nested_struct(tmp_path):
-    table = pa.table(
-        {
-            "x": pa.array(
-                [
-                    {
-                        "a": 1,
-                        "json": {"b": "hello", "x": b"abc"},
-                    },
-                    {
-                        "a": 24,
-                        "json": {"b": "world", "x": b"def"},
-                    },
-                ]
-            )
-        }
-    )
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-
-    ds = tf.data.Dataset.from_lance(
-        dataset,
-        batch_size=8,
-    )
-
-    for batch in ds:
-        tf.debugging.assert_equal(batch["x"]["a"], tf.constant([1, 24], dtype=tf.int64))
-        tf.debugging.assert_equal(
-            batch["x"]["json"]["b"], tf.constant(["hello", "world"])
-        )
-        tf.debugging.assert_equal(
-            batch["x"]["json"]["x"], tf.constant([b"abc", b"def"])
-        )
-
-
-def test_tensor(tmp_path):
-    arr = np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]], dtype=np.float32)
-    table = pa.table({"x": pa.FixedShapeTensorArray.from_numpy_ndarray(arr)})
-
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-    ds = tf.data.Dataset.from_lance(dataset)
-
-    for batch in ds:
-        assert batch["x"].shape == (2, 2, 3)
-        assert batch["x"].dtype == tf.float32
-        assert batch["x"].numpy().tolist() == arr.tolist()
-
-
-def test_image_types(tmp_path):
-    path = [os.path.join(os.path.dirname(__file__), "images/1.png")]
-    uris = ImageArray.from_array(path * 3)
-    encoded_images = uris.read_uris()
-    tensors = encoded_images.to_tensor()
-    table = pa.table(
-        {
-            "uris": uris,
-            "encoded_images": encoded_images,
-            "tensor_images": tensors,
-        }
-    )
-
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-    ds = tf.data.Dataset.from_lance(dataset)
-
-    for batch in ds:
-        assert batch["uris"].shape == (3,)
-        assert batch["uris"].dtype == tf.string
-        assert batch["uris"].numpy().astype("str").tolist() == uris.tolist()
-
-        assert batch["encoded_images"].shape == (3,)
-        assert batch["encoded_images"].dtype == tf.string
-        assert batch["encoded_images"].numpy().tolist() == encoded_images.tolist()
-
-        assert batch["tensor_images"].shape == (3, 1, 1, 4)
-        assert batch["tensor_images"].dtype == tf.uint8
-        assert batch["tensor_images"].numpy().tolist() == tensors.to_numpy().tolist()
diff --git a/python/python/tests/test_vector.py b/python/python/tests/test_vector.py
index 4ea4e7d425e..12caec5f0d5 100644
--- a/python/python/tests/test_vector.py
+++ b/python/python/tests/test_vector.py
@@ -5,7 +5,12 @@
 import numpy as np
 import pyarrow as pa
 import pytest
-from lance.vector import hamming_clustering_for_sample, vec_to_table
+from lance.vector import (
+    get_ivf_partition_info,
+    hamming_clustering_for_ivf_partition,
+    hamming_clustering_for_sample,
+    vec_to_table,
+)
 
 
 def test_dict():
@@ -150,21 +155,26 @@ def test_binary_vectors_invalid_metric(tmp_path):
 
 
 def _hash_table(hashes):
-    """Build a table with a ``hash`` column of FixedSizeList<UInt8, 8>.
+    """Build a table with a ``hash`` column of FixedSizeList<UInt8, N>.
 
-    ``hashes`` is a list of 8-byte sequences, one per row.
+    ``hashes`` is a list of byte sequences, one per row. The byte width must
+    be a positive multiple of 8.
     """
+    byte_width = len(hashes[0])
+    assert byte_width > 0 and byte_width % 8 == 0
+    assert all(len(row) == byte_width for row in hashes)
     flat = [byte for row in hashes for byte in row]
     values = pa.FixedSizeListArray.from_arrays(
-        pa.array(flat, type=pa.uint8()), list_size=8
+        pa.array(flat, type=pa.uint8()), list_size=byte_width
     )
     return pa.Table.from_arrays([values], names=["hash"])
 
 
-def test_hamming_clustering_for_sample(tmp_path):
-    hash_a = [0, 0, 0, 0, 0, 0, 0, 0]
-    hash_b = [255, 0, 0, 0, 0, 0, 0, 0]  # 8 bits from hash_a
-    hash_c = [1, 2, 3, 4, 5, 6, 7, 8]  # far from both
+@pytest.mark.parametrize("byte_width", [8, 16])
+def test_hamming_clustering_for_sample(tmp_path, byte_width):
+    hash_a = [0] * byte_width
+    hash_b = [0] * (byte_width - 8) + [255] + [0] * 7  # 8 bits from hash_a
+    hash_c = list(range(1, byte_width + 1))  # far from both
     # Rows 0,1,2 share hash_a; rows 3,4 share hash_b; row 5 is unique.
     table = _hash_table([hash_a, hash_a, hash_a, hash_b, hash_b, hash_c])
     dataset = lance.write_dataset(table, tmp_path / "hashes")
@@ -182,3 +192,89 @@ def test_hamming_clustering_for_sample(tmp_path):
     }
     # Singleton row 5 is not emitted as a cluster.
     assert clusters == {0: [1, 2], 3: [4]}
+
+
+@pytest.mark.parametrize("byte_width", [8, 16])
+def test_hamming_clustering_multi_segment(tmp_path, byte_width):
+    mask = (1 << 64) - 1
+
+    def hash_bytes(value):
+        if byte_width == 8:
+            lanes = [(value * 0x9E3779B97F4A7C15) & mask]
+        else:
+            # Adjacent logical values share the first 64-bit lane and differ in
+            # later lanes, so threshold-0 clustering must compare every lane.
+            lanes = [
+                ((value // 2) * 0x9E3779B97F4A7C15) & mask,
+                ((value * 0xD6E8FEB86659FD93) ^ 0xA5A5A5A5A5A5A5A5) & mask,
+            ]
+        return [
+            byte for lane_value in lanes for byte in lane_value.to_bytes(8, "little")
+        ]
+
+    # 25 distinct hash values, two copies each; the same table is written to
+    # fragment 0 and appended as fragment 1.
+    values = [i // 2 for i in range(50)]
+    table = _hash_table([hash_bytes(value) for value in values])
+    dataset = lance.write_dataset(table, tmp_path / "hashes")
+    dataset.create_index(
+        "hash", index_type="IVF_FLAT", num_partitions=4, metric="hamming"
+    )
+    dataset = lance.write_dataset(table, tmp_path / "hashes", mode="append")
+    # Optimizing with merge disabled creates a delta segment for fragment 1.
+    dataset.optimize.optimize_indices(num_indices_to_merge=0)
+
+    index = dataset.describe_indices()[0]
+    assert len(index.segments) == 2
+
+    infos = get_ivf_partition_info(dataset, index.name)
+    assert sum(info["size"] for info in infos) == 100
+
+    # All four copies of each value cluster together across both fragments.
+    frag1_start = 1 << 32
+    clusters = []
+    for info in infos:
+        result = hamming_clustering_for_ivf_partition(
+            dataset, index.name, info["partition_id"], 0
+        ).read_all()
+        clusters.extend(
+            zip(
+                result["representative"].to_pylist(),
+                result["duplicates"].to_pylist(),
+            )
+        )
+    assert len(clusters) == 25
+    for representative, duplicates in clusters:
+        assert representative < frag1_start
+        assert len(duplicates) == 3
+        assert any(dup >= frag1_start for dup in duplicates)
+
+    # Selecting the fragment-0 segment reproduces the single-segment scope.
+    first_segment = next(
+        segment for segment in index.segments if segment.fragment_ids == {0}
+    )
+    infos = get_ivf_partition_info(
+        dataset, index.name, index_segments=[first_segment.uuid]
+    )
+    assert sum(info["size"] for info in infos) == 50
+    num_selected_clusters = 0
+    for info in infos:
+        result = hamming_clustering_for_ivf_partition(
+            dataset,
+            index.name,
+            info["partition_id"],
+            0,
+            index_segments=[first_segment.uuid],
+        ).read_all()
+        for duplicates in result["duplicates"].to_pylist():
+            num_selected_clusters += 1
+            assert duplicates == [dup for dup in duplicates if dup < frag1_start]
+            assert len(duplicates) == 1
+    assert num_selected_clusters == 25
+
+    with pytest.raises(ValueError, match="invalid index segment uuid"):
+        get_ivf_partition_info(dataset, index.name, index_segments=["not-a-uuid"])
+    with pytest.raises(TypeError, match="str or uuid.UUID"):
+        get_ivf_partition_info(dataset, index.name, index_segments=[123])
+    with pytest.raises(TypeError, match="not a single"):
+        get_ivf_partition_info(dataset, index.name, index_segments=first_segment.uuid)
diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py
index fa2c4047cd0..e76f7069298 100644
--- a/python/python/tests/test_vector_index.py
+++ b/python/python/tests/test_vector_index.py
@@ -222,6 +222,46 @@ def test_batch_flat_query_matches_repeated_single_queries(dataset, queries):
     )
 
 
+@pytest.mark.parametrize("metric", ["l2", "cosine"])
+@pytest.mark.parametrize("query_count", [3, 1], ids=["three_queries", "single_query"])
+def test_batch_indexed_query_matches_repeated_single_queries(
+    dataset, metric, query_count
+):
+    indexed = dataset.create_index(
+        "vector",
+        index_type="IVF_PQ",
+        num_partitions=4,
+        num_sub_vectors=16,
+        metric=metric,
+    )
+    # Give the query vectors deliberately different magnitudes: a cosine batch
+    # that normalized the whole concatenated key by one global norm would scale
+    # them unequally and diverge from per-query single search.
+    scales = np.linspace(0.1, 10.0, query_count).reshape(-1, 1)
+    queries = (np.random.randn(query_count, 128) * scales).astype(np.float32)
+    k = 5
+
+    # nprobes covers every partition so the shared-scan batch path and the
+    # repeated single-query path search the same partitions deterministically.
+    nearest_kwargs = {"use_index": True, "nprobes": 4}
+    batch = indexed.to_table(
+        columns=["id"],
+        nearest={"column": "vector", "q": queries, "k": k, **nearest_kwargs},
+    )
+
+    assert batch.column_names == ["query_index", "id", "_distance"]
+    assert batch["query_index"].to_pylist() == sum(
+        [[i] * k for i in range(query_count)], []
+    )
+
+    _assert_batch_matches_single_queries(
+        indexed,
+        queries,
+        k=k,
+        nearest_kwargs=nearest_kwargs,
+    )
+
+
 def _assert_batch_matches_single_queries(ds, queries, k, nearest_kwargs):
     batch = ds.to_table(
         columns=["id"],
@@ -2468,7 +2508,7 @@ def test_vector_index_distance_range(tmp_path):
     assert np.all(index_distances >= distance_range[0]) and np.all(
         index_distances < distance_range[1]
     )
-    assert np.allclose(brute_distances, index_distances, rtol=0.0, atol=0.0)
+    assert np.allclose(brute_distances, index_distances, rtol=1e-5, atol=0.0)
 
 
 # =============================================================================
diff --git a/python/python/tests/torch_tests/test_torch_kmeans.py b/python/python/tests/torch_tests/test_torch_kmeans.py
index edfb0a0329a..96bbf76c51f 100644
--- a/python/python/tests/torch_tests/test_torch_kmeans.py
+++ b/python/python/tests/torch_tests/test_torch_kmeans.py
@@ -10,15 +10,16 @@
 
 torch = pytest.importorskip("torch")
 
-from lance.torch import preferred_device  # noqa: E402
 from lance.torch.kmeans import KMeans  # noqa: E402
 from lance.vector import train_ivf_centroids_on_accelerator  # noqa: E402
 
 
-@pytest.mark.skip(reason="flaky")
 def test_kmeans():
     arr = np.array(range(128)).reshape(-1, 8).astype(np.float32)
-    kmeans = KMeans(4, device="cpu")
+    # These duplicate centroids reproduce the empty clusters that made this test
+    # depend on random initialization before empty-cluster recovery was fixed.
+    centroids = torch.from_numpy(arr[[5, 5, 13, 13]])
+    kmeans = KMeans(4, centroids=centroids, device="cpu")
     kmeans.fit(arr)
 
     cluster_ids = kmeans.transform(arr)
@@ -27,7 +28,6 @@ def test_kmeans():
     assert len(cnts) == 4  # all cluster has data
 
 
-@pytest.mark.skip(reason="TODO: async dataset hangs on github CI")
 def test_torch_kmeans_accept_torch_device(tmp_path: Path):
     values = pa.array(np.array(range(128)).astype(np.float32))
     arr = pa.FixedSizeListArray.from_arrays(values, 8)
@@ -39,7 +39,7 @@ def test_torch_kmeans_accept_torch_device(tmp_path: Path):
         "vector",
         2,
         metric_type="L2",
-        accelerator=preferred_device(),
+        accelerator=torch.device("cpu"),
     )
 
 
diff --git a/python/src/blob.rs b/python/src/blob.rs
index 82e8a01ae8a..1f03cd9abe5 100644
--- a/python/src/blob.rs
+++ b/python/src/blob.rs
@@ -2,18 +2,166 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use crate::{error::PythonErrorExt, rt};
-use arrow::pyarrow::ToPyArrow;
+use arrow::{
+    array::{Array, ArrayRef, GenericBinaryArray, OffsetSizeTrait, cast::AsArray, make_array},
+    pyarrow::{FromPyArrow, ToPyArrow},
+};
+use arrow_data::ArrayData;
+use arrow_schema::{DataType, Field};
 use bytes::Bytes;
 use lance::{
     BlobDescriptor, BlobDescriptorArrayBuilder, BlobRange, DedicatedBlobWriter, PackedBlobWriter,
 };
 use pyo3::{
-    Bound, PyResult,
-    exceptions::PyValueError,
+    Bound, PyErr, PyResult,
+    exceptions::{PyRuntimeError, PyValueError},
     pyclass, pymethods,
-    types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule},
+    types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule, PyTypeMethods},
+};
+use std::{
+    borrow::Cow,
+    sync::{Arc, Mutex},
 };
-use std::sync::Arc;
+
+fn with_writer<W, R>(
+    inner: &Mutex<Option<W>>,
+    writer_name: &str,
+    operation: impl FnOnce(&W) -> R,
+) -> PyResult<R> {
+    let guard = inner.lock().map_err(|_| poisoned_writer(writer_name))?;
+    let writer = guard.as_ref().ok_or_else(|| finished_writer(writer_name))?;
+    Ok(operation(writer))
+}
+
+fn writer_mut<'a, W>(inner: &'a mut Mutex<Option<W>>, writer_name: &str) -> PyResult<&'a mut W> {
+    inner
+        .get_mut()
+        .map_err(|_| poisoned_writer(writer_name))?
+        .as_mut()
+        .ok_or_else(|| finished_writer(writer_name))
+}
+
+fn take_writer<W>(inner: &mut Mutex<Option<W>>, writer_name: &str) -> PyResult<W> {
+    inner
+        .get_mut()
+        .map_err(|_| poisoned_writer(writer_name))?
+        .take()
+        .ok_or_else(|| finished_writer(writer_name))
+}
+
+fn finished_writer(writer_name: &str) -> PyErr {
+    PyValueError::new_err(format!("{writer_name} is already finished"))
+}
+
+fn poisoned_writer(writer_name: &str) -> PyErr {
+    PyRuntimeError::new_err(format!("{writer_name} lock is poisoned"))
+}
+
+/// Reconstruct the PyArrow equivalent of [`BlobDescriptorArrayBuilder::field`].
+///
+/// Arrow's array bridge does not carry the enclosing extension field, so this
+/// rebuilds the canonical six nullable blob-v2 children and
+/// `ARROW:extension:name = lance.blob.v2` metadata.
+fn descriptor_field_to_pyarrow<'py>(
+    field: &Field,
+    py: pyo3::Python<'py>,
+) -> PyResult<Bound<'py, PyAny>> {
+    let pyarrow = PyModule::import(py, "pyarrow")?;
+    let child_fields = PyList::empty(py);
+    for (name, type_fn) in [
+        ("kind", "uint8"),
+        ("data", "large_binary"),
+        ("uri", "utf8"),
+        ("blob_id", "uint32"),
+        ("blob_size", "uint64"),
+        ("position", "uint64"),
+    ] {
+        let data_type = pyarrow.getattr(type_fn)?.call0()?;
+        let child = pyarrow.call_method1("field", (name, data_type, true))?;
+        child_fields.append(child)?;
+    }
+    let data_type = pyarrow.call_method1("struct", (child_fields,))?;
+    let metadata = PyDict::new(py);
+    metadata.set_item("ARROW:extension:name", "lance.blob.v2")?;
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("nullable", field.is_nullable())?;
+    kwargs.set_item("metadata", metadata)?;
+    pyarrow.call_method("field", (field.name().as_str(), data_type), Some(&kwargs))
+}
+
+/// Normalize inputs accepted by [`PyPackedBlobWriter::write_blobs`] into Arrow arrays.
+///
+/// BinaryArray, LargeBinaryArray, and ChunkedArray values of either binary type
+/// are accepted. Chunk boundaries, nulls, and empty values remain in the arrays;
+/// each row is later passed to the core writer as an optional byte slice.
+fn extract_blob_payloads(payloads: &Bound<'_, PyAny>) -> PyResult<Vec<ArrayRef>> {
+    match ArrayData::from_pyarrow_bound(payloads) {
+        Ok(data) => Ok(vec![validated_blob_payload(data, None)?]),
+        Err(_) => {
+            let pyarrow = PyModule::import(payloads.py(), "pyarrow")?;
+            let chunked_array_type = pyarrow.getattr("ChunkedArray")?;
+            if !payloads.is_instance(&chunked_array_type)? {
+                return Err(PyValueError::new_err(format!(
+                    "payloads must be a pyarrow BinaryArray, LargeBinaryArray, or ChunkedArray, got {}",
+                    payloads.get_type().name()?
+                )));
+            }
+
+            let chunked_data_type = DataType::from_pyarrow_bound(&payloads.getattr("type")?)?;
+            if !matches!(chunked_data_type, DataType::Binary | DataType::LargeBinary) {
+                return Err(PyValueError::new_err(format!(
+                    "Packed blob payloads must have Arrow type Binary or LargeBinary, got {chunked_data_type}"
+                )));
+            }
+
+            let chunks = payloads.getattr("chunks")?;
+            let mut arrays = Vec::with_capacity(chunks.len()?);
+            for (chunk_index, chunk) in chunks.try_iter()?.enumerate() {
+                let data = ArrayData::from_pyarrow_bound(&chunk?)?;
+                arrays.push(validated_blob_payload(data, Some(chunk_index))?);
+            }
+            Ok(arrays)
+        }
+    }
+}
+
+fn validated_blob_payload(data: ArrayData, chunk_index: Option<usize>) -> PyResult<ArrayRef> {
+    let context = chunk_index
+        .map(|index| format!("Packed blob payload chunk {index}"))
+        .unwrap_or_else(|| "Packed blob payload array".to_string());
+    if !matches!(data.data_type(), DataType::Binary | DataType::LargeBinary) {
+        return Err(PyValueError::new_err(format!(
+            "{context} must have Arrow type Binary or LargeBinary, got {}",
+            data.data_type()
+        )));
+    }
+    if data.is_empty() {
+        // PyArrow may export an empty slice without the values preceding its
+        // nonzero first offset. Normalize it because an empty array never
+        // observes those buffers, and Arrow validation would reject the slice.
+        return Ok(make_array(ArrayData::new_empty(data.data_type())));
+    }
+    data.validate_full().map_err(|error| {
+        PyValueError::new_err(format!("{context} contains invalid Arrow data: {error}"))
+    })?;
+    Ok(make_array(data))
+}
+
+/// Stream one Arrow binary array into the core writer as zero-copy row slices.
+///
+/// Null rows become `None` so the core writer records null descriptors, keeping
+/// its output row-aligned with the input.
+async fn write_binary_payloads<O: OffsetSizeTrait>(
+    writer: &mut PackedBlobWriter,
+    payloads: &GenericBinaryArray<O>,
+) -> PyResult<()> {
+    writer
+        .write_packed_blobs(
+            (0..payloads.len()).map(|row| payloads.is_valid(row).then(|| payloads.value(row))),
+        )
+        .await
+        .infer_error()
+}
 
 #[pyclass(name = "BlobDescriptor", skip_from_py_object)]
 #[derive(Clone)]
@@ -61,31 +209,7 @@ impl PyBlobDescriptorArrayBuilder {
 
     #[getter]
     pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult<Bound<'py, PyAny>> {
-        let pyarrow = PyModule::import(py, "pyarrow")?;
-        let child_fields = PyList::empty(py);
-        for (name, type_fn) in [
-            ("kind", "uint8"),
-            ("data", "large_binary"),
-            ("uri", "utf8"),
-            ("blob_id", "uint32"),
-            ("blob_size", "uint64"),
-            ("position", "uint64"),
-        ] {
-            let data_type = pyarrow.getattr(type_fn)?.call0()?;
-            let child = pyarrow.call_method1("field", (name, data_type, true))?;
-            child_fields.append(child)?;
-        }
-        let data_type = pyarrow.call_method1("struct", (child_fields,))?;
-        let metadata = PyDict::new(py);
-        metadata.set_item("ARROW:extension:name", "lance.blob.v2")?;
-        let kwargs = PyDict::new(py);
-        kwargs.set_item("nullable", self.field.is_nullable())?;
-        kwargs.set_item("metadata", metadata)?;
-        pyarrow.call_method(
-            "field",
-            (self.field.name().as_str(), data_type),
-            Some(&kwargs),
-        )
+        descriptor_field_to_pyarrow(&self.field, py)
     }
 
     pub fn extend_packed(
@@ -150,9 +274,10 @@ impl PyBlobDescriptorArrayBuilder {
     }
 }
 
-#[pyclass(name = "PackedBlobWriter", skip_from_py_object, unsendable)]
+#[pyclass(name = "PackedBlobWriter", skip_from_py_object)]
 pub struct PyPackedBlobWriter {
-    inner: Option<PackedBlobWriter>,
+    field: Option<Field>,
+    inner: Mutex<Option<PackedBlobWriter>>,
 }
 
 impl PyPackedBlobWriter {
@@ -165,19 +290,22 @@ impl PyPackedBlobWriter {
             PackedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id)
                 .await
                 .infer_error()?;
-        Ok(Self { inner: Some(inner) })
+        Ok(Self {
+            field: None,
+            inner: Mutex::new(Some(inner)),
+        })
     }
 
-    fn inner(&self) -> PyResult<&PackedBlobWriter> {
-        self.inner
-            .as_ref()
-            .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished"))
+    fn with_inner<R>(&self, operation: impl FnOnce(&PackedBlobWriter) -> R) -> PyResult<R> {
+        with_writer(&self.inner, "PackedBlobWriter", operation)
     }
 
     fn inner_mut(&mut self) -> PyResult<&mut PackedBlobWriter> {
-        self.inner
-            .as_mut()
-            .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished"))
+        writer_mut(&mut self.inner, "PackedBlobWriter")
+    }
+
+    fn take_inner(&mut self) -> PyResult<PackedBlobWriter> {
+        take_writer(&mut self.inner, "PackedBlobWriter")
     }
 }
 
@@ -185,32 +313,142 @@ impl PyPackedBlobWriter {
 impl PyPackedBlobWriter {
     #[getter]
     pub fn blob_id(&self) -> PyResult<u32> {
-        Ok(self.inner()?.blob_id())
+        self.with_inner(PackedBlobWriter::blob_id)
     }
 
     #[getter]
     pub fn path(&self) -> PyResult<String> {
-        Ok(self.inner()?.path().to_string())
+        self.with_inner(|writer| writer.path().to_string())
+    }
+
+    /// The descriptor field associated with the array returned by
+    /// :meth:`finish_array`.
+    ///
+    /// The field uses the name passed to ``finish_array`` and carries the
+    /// ``lance.blob.v2`` extension metadata. It is available only after
+    /// ``finish_array`` succeeds; accessing it earlier raises ``ValueError``.
+    #[getter]
+    pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let field = self.field.as_ref().ok_or_else(|| {
+            PyValueError::new_err("PackedBlobWriter field is available after finish_array")
+        })?;
+        descriptor_field_to_pyarrow(field, py)
     }
 
-    pub fn write_blob(&mut self, data: Vec<u8>) -> PyResult<()> {
-        rt().block_on(None, self.inner_mut()?.write_blob(data))?
+    /// Append one packed blob.
+    ///
+    /// Python ``bytes`` are borrowed without copying. Other compatible byte
+    /// sequences use owned storage for the duration of the write.
+    pub fn write_blob(&mut self, data: Cow<'_, [u8]>) -> PyResult<()> {
+        rt().block_on(None, self.inner_mut()?.write_blob(data.as_ref()))?
             .infer_error()
     }
 
+    /// Append a batch of packed blob payloads.
+    ///
+    /// Parameters
+    /// ----------
+    /// payloads : pyarrow.BinaryArray, pyarrow.LargeBinaryArray, or pyarrow.ChunkedArray
+    ///     A binary Arrow array. Every chunk of a chunked array must be binary.
+    ///     Each input row produces one descriptor row, in order, across chunks
+    ///     and repeated calls. Null rows produce null descriptors; empty but
+    ///     non-null byte strings produce valid zero-length blobs.
+    ///
+    /// Examples
+    /// --------
+    /// >>> import pyarrow as pa
+    /// >>> payloads = pa.array([b"first", None, b""], type=pa.large_binary())
+    /// >>> writer.write_blobs(payloads)
+    /// >>> descriptors = writer.finish_array("blob")
+    /// >>> len(descriptors)
+    /// 3
+    pub fn write_blobs(&mut self, payloads: &Bound<'_, PyAny>) -> PyResult<()> {
+        let payloads = extract_blob_payloads(payloads)?;
+        let result = {
+            let writer = self.inner_mut()?;
+            rt().block_on(None, async {
+                for payloads in payloads {
+                    match payloads.data_type() {
+                        DataType::Binary => {
+                            write_binary_payloads(writer, payloads.as_binary::<i32>()).await?
+                        }
+                        DataType::LargeBinary => {
+                            write_binary_payloads(writer, payloads.as_binary::<i64>()).await?
+                        }
+                        data_type => {
+                            return Err(PyValueError::new_err(format!(
+                                "Packed blob payloads must have Arrow type Binary or LargeBinary, got {data_type}"
+                            )));
+                        }
+                    }
+                }
+                Ok(())
+            })
+        };
+        match result {
+            Ok(result) => result,
+            Err(error) => {
+                // KeyboardInterrupt drops the async batch future. Remove the core
+                // writer as well so RAII cleanup runs and a completed prefix cannot
+                // be reused as a new batch.
+                self.take_inner()?;
+                Err(error)
+            }
+        }
+    }
+
     pub fn finish(&mut self) -> PyResult<Vec<PyBlobDescriptor>> {
-        let inner = self
-            .inner
-            .take()
-            .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished"))?;
+        let inner = self.take_inner()?;
         let values = rt().block_on(None, inner.finish())?.infer_error()?;
         Ok(values.into_iter().map(Into::into).collect())
     }
+
+    /// Finish the upload and return its blob descriptors as a PyArrow array.
+    ///
+    /// The returned ``pyarrow.StructArray`` has one row per payload previously
+    /// passed to :meth:`write_blob` or :meth:`write_blobs`. The writer is consumed
+    /// by this call. After it succeeds, :attr:`field` returns the matching
+    /// extension field with ``field_name`` as its name.
+    ///
+    /// Parameters
+    /// ----------
+    /// field_name : str
+    ///     Name for the descriptor field exposed by :attr:`field`.
+    ///
+    /// Returns
+    /// -------
+    /// pyarrow.StructArray
+    ///     Row-aligned blob descriptors, including null rows from bulk input.
+    ///
+    /// Examples
+    /// --------
+    /// >>> import pyarrow as pa
+    /// >>> writer.write_blobs(pa.array([b"value", None]))
+    /// >>> descriptors = writer.finish_array("payload")
+    /// >>> descriptors.is_null().to_pylist()
+    /// [False, True]
+    /// >>> writer.field.name
+    /// 'payload'
+    pub fn finish_array<'py>(
+        &mut self,
+        py: pyo3::Python<'py>,
+        field_name: String,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.take_inner()?;
+        let values = rt().block_on(None, inner.finish())?.infer_error()?;
+        let mut builder = BlobDescriptorArrayBuilder::new(field_name);
+        builder.extend(values).infer_error()?;
+        let column = builder.finish().infer_error()?;
+        let (field, array) = column.into_parts();
+        let array = array.to_data().to_pyarrow(py)?;
+        self.field = Some(field);
+        Ok(array)
+    }
 }
 
-#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object, unsendable)]
+#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object)]
 pub struct PyDedicatedBlobWriter {
-    inner: Option<DedicatedBlobWriter>,
+    inner: Mutex<Option<DedicatedBlobWriter>>,
 }
 
 impl PyDedicatedBlobWriter {
@@ -223,19 +461,21 @@ impl PyDedicatedBlobWriter {
             DedicatedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id)
                 .await
                 .infer_error()?;
-        Ok(Self { inner: Some(inner) })
+        Ok(Self {
+            inner: Mutex::new(Some(inner)),
+        })
     }
 
-    fn inner(&self) -> PyResult<&DedicatedBlobWriter> {
-        self.inner
-            .as_ref()
-            .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished"))
+    fn with_inner<R>(&self, operation: impl FnOnce(&DedicatedBlobWriter) -> R) -> PyResult<R> {
+        with_writer(&self.inner, "DedicatedBlobWriter", operation)
     }
 
     fn inner_mut(&mut self) -> PyResult<&mut DedicatedBlobWriter> {
-        self.inner
-            .as_mut()
-            .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished"))
+        writer_mut(&mut self.inner, "DedicatedBlobWriter")
+    }
+
+    fn take_inner(&mut self) -> PyResult<DedicatedBlobWriter> {
+        take_writer(&mut self.inner, "DedicatedBlobWriter")
     }
 }
 
@@ -243,12 +483,12 @@ impl PyDedicatedBlobWriter {
 impl PyDedicatedBlobWriter {
     #[getter]
     pub fn blob_id(&self) -> PyResult<u32> {
-        Ok(self.inner()?.blob_id())
+        self.with_inner(DedicatedBlobWriter::blob_id)
     }
 
     #[getter]
     pub fn path(&self) -> PyResult<String> {
-        Ok(self.inner()?.path().to_string())
+        self.with_inner(|writer| writer.path().to_string())
     }
 
     pub fn write(&mut self, data: Vec<u8>) -> PyResult<()> {
@@ -257,10 +497,7 @@ impl PyDedicatedBlobWriter {
     }
 
     pub fn finish(&mut self) -> PyResult<PyBlobDescriptor> {
-        let inner = self
-            .inner
-            .take()
-            .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished"))?;
+        let inner = self.take_inner()?;
         let value = rt().block_on(None, inner.finish())?.infer_error()?;
         Ok(value.into())
     }
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index 350428c89aa..d974878f229 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -70,6 +70,7 @@ use lance_core::datatypes::BlobHandling;
 use lance_datafusion::utils::reader_to_stream;
 use lance_encoding::decoder::DecoderConfig;
 use lance_file::reader::FileReaderOptions;
+use lance_index::scalar::inverted::InvertedListFormatVersion;
 use lance_index::scalar::inverted::query::Occur;
 use lance_index::scalar::inverted::query::{
     BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
@@ -78,7 +79,6 @@ use lance_index::{
     FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
     progress::{IndexBuildProgress, NoopIndexBuildProgress},
-    scalar::inverted::InvertedListFormatVersion,
     scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams},
     vector::{
         ApproxMode, DEFAULT_QUERY_PARALLELISM, Query as VectorQuery,
@@ -125,13 +125,44 @@ const DEFAULT_NPROBES: usize = 1;
 const LANCE_COMMIT_MESSAGE_KEY: &str = "__lance_commit_message";
 const INDEX_PROGRESS_QUEUE_SIZE: usize = 1024;
 
-fn read_blobs_to_python(
-    py: Python<'_>,
-    blobs: Vec<lance::dataset::ReadBlob>,
-) -> Vec<(u64, Py<PyBytes>)> {
+type PyBlobBytes = Option<Py<PyBytes>>;
+type PyReadBlob = (u64, PyBlobBytes);
+type PyReadBlobRange = (usize, u64, PyBlobBytes);
+
+fn read_blobs_to_python(py: Python<'_>, blobs: Vec<lance::dataset::ReadBlob>) -> Vec<PyReadBlob> {
     blobs
         .into_iter()
-        .map(|blob| (blob.row_address, PyBytes::new(py, &blob.data).unbind()))
+        .map(|blob| {
+            (
+                blob.row_address,
+                blob.data.map(|data| PyBytes::new(py, &data).unbind()),
+            )
+        })
+        .collect()
+}
+
+fn read_blob_ranges_to_python(
+    py: Python<'_>,
+    ranges: Vec<lance::dataset::ReadBlobRange>,
+) -> Vec<PyReadBlobRange> {
+    ranges
+        .into_iter()
+        .map(|range| {
+            (
+                range.request_index,
+                range.row_address,
+                range.data.map(|data| PyBytes::new(py, &data).unbind()),
+            )
+        })
+        .collect()
+}
+
+fn blob_range_requests_from_tuples(
+    requests: Vec<(u64, u64, u64)>,
+) -> Vec<lance::dataset::BlobRangeRequest> {
+    requests
+        .into_iter()
+        .map(|(row, offset, length)| lance::dataset::BlobRangeRequest::new(row, offset, length))
         .collect()
 }
 
@@ -149,6 +180,20 @@ fn configure_read_blobs_builder(
     builder
 }
 
+fn configure_read_blob_ranges_builder(
+    mut builder: lance::dataset::ReadBlobRangesBuilder,
+    io_buffer_size: Option<u64>,
+    preserve_order: Option<bool>,
+) -> lance::dataset::ReadBlobRangesBuilder {
+    if let Some(bytes) = io_buffer_size {
+        builder = builder.with_io_buffer_size_bytes(bytes);
+    }
+    if let Some(preserve) = preserve_order {
+        builder = builder.preserve_order(preserve);
+    }
+    builder
+}
+
 fn stats_log_interval_from_millis(ms: u64) -> Option<std::time::Duration> {
     if ms == 0 {
         None
@@ -164,7 +209,6 @@ fn stats_log_interval_from_millis(ms: u64) -> Option<std::time::Duration> {
 #[allow(clippy::too_many_arguments)]
 fn writer_config_from_kwargs(
     durable_write: Option<bool>,
-    sync_indexed_write: Option<bool>,
     max_wal_buffer_size: Option<usize>,
     max_wal_flush_interval_ms: Option<u64>,
     max_memtable_size: Option<usize>,
@@ -172,8 +216,6 @@ fn writer_config_from_kwargs(
     max_memtable_batches: Option<usize>,
     max_unflushed_memtable_bytes: Option<usize>,
     manifest_scan_batch_size: Option<usize>,
-    async_index_buffer_rows: Option<usize>,
-    async_index_interval_ms: Option<u64>,
     backpressure_log_interval_ms: Option<u64>,
     stats_log_interval_ms: Option<u64>,
     hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -186,10 +228,6 @@ fn writer_config_from_kwargs(
         config = config.with_durable_write(v);
         any = true;
     }
-    if let Some(v) = sync_indexed_write {
-        config = config.with_sync_indexed_write(v);
-        any = true;
-    }
     if let Some(v) = max_wal_buffer_size {
         config = config.with_max_wal_buffer_size(v);
         any = true;
@@ -218,14 +256,6 @@ fn writer_config_from_kwargs(
         config = config.with_manifest_scan_batch_size(v);
         any = true;
     }
-    if let Some(v) = async_index_buffer_rows {
-        config = config.with_async_index_buffer_rows(v);
-        any = true;
-    }
-    if let Some(v) = async_index_interval_ms {
-        config = config.with_async_index_interval(Duration::from_millis(v));
-        any = true;
-    }
     if let Some(v) = backpressure_log_interval_ms {
         config = config.with_backpressure_log_interval(Duration::from_millis(v));
         any = true;
@@ -456,6 +486,60 @@ impl MergeInsertBuilder {
         Ok((PyLance(transaction), stats))
     }
 
+    /// Execute the merge insert from fully-materialized data.
+    ///
+    /// The data is read into memory and wrapped in an in-memory table, so retries
+    /// never spill to disk and the source's statistics drive the join. Callers
+    /// should only route in-memory inputs (e.g. a `pa.Table`) here.
+    pub fn execute_batches(&mut self, new_data: &Bound<PyAny>) -> PyResult<Py<PyAny>> {
+        let py = new_data.py();
+        let reader = convert_reader(new_data)?;
+        let batches = reader
+            .collect::<std::result::Result<Vec<RecordBatch>, _>>()
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        let job = self
+            .builder
+            .try_build()
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        let (new_dataset, stats) = rt()
+            .spawn(Some(py), job.execute_batches(batches))?
+            .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?;
+
+        let dataset = self.dataset.bind(py);
+        dataset.borrow_mut().ds = new_dataset;
+
+        Ok(Self::build_stats(&stats, py)?.into())
+    }
+
+    /// [`Self::execute_batches`] without committing; returns the transaction.
+    pub fn execute_uncommitted_batches<'a>(
+        &mut self,
+        new_data: &Bound<'a, PyAny>,
+    ) -> PyResult<(PyLance<Transaction>, Bound<'a, PyDict>)> {
+        let py = new_data.py();
+        let reader = convert_reader(new_data)?;
+        let batches = reader
+            .collect::<std::result::Result<Vec<RecordBatch>, _>>()
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        let job = self
+            .builder
+            .try_build()
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        let UncommittedMergeInsert {
+            transaction, stats, ..
+        } = rt()
+            .spawn(Some(py), job.execute_uncommitted_batches(batches))?
+            .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?;
+
+        let stats = Self::build_stats(&stats, py)?;
+
+        Ok((PyLance(transaction), stats))
+    }
+
     #[pyo3(signature=(schema = None, verbose = false))]
     pub fn explain_plan(
         &mut self,
@@ -487,22 +571,21 @@ impl MergeInsertBuilder {
             .map_err(|err| PyIOError::new_err(err.to_string()))
     }
 
-    /// Mark MemWAL generations as merged into the base table.
+    /// Mark MemWAL SSTables as compacted into the base table.
     ///
-    /// Call this when executing a merge_insert that incorporates MemWAL
-    /// flushed generation data. This updates the MemWAL generation tracking
-    /// to prevent duplicate merges.
-    pub fn mark_generations_as_merged<'a>(
+    /// Call this when executing a merge_insert that compacts MemWAL SSTables.
+    /// This updates MemWAL compaction progress to prevent duplicate compactions.
+    pub fn mark_sstables_as_compacted<'a>(
         mut slf: PyRefMut<'a, Self>,
-        generations: Vec<Bound<'a, crate::mem_wal::PyMergedGeneration>>,
+        sstables: Vec<Bound<'a, crate::mem_wal::PyCompactedSsTable>>,
     ) -> PyResult<PyRefMut<'a, Self>> {
-        use lance_index::mem_wal::MergedGeneration;
+        use lance_index::mem_wal::CompactedSsTable;
 
-        let gens: Vec<MergedGeneration> = generations
+        let compacted_sstables: Vec<CompactedSsTable> = sstables
             .iter()
-            .map(|g| g.borrow().to_lance())
+            .map(|sstable| sstable.borrow().to_lance())
             .collect::<PyResult<_>>()?;
-        slf.builder.mark_generations_as_merged(gens);
+        slf.builder.mark_sstables_as_compacted(compacted_sstables);
         Ok(slf)
     }
 }
@@ -524,8 +607,10 @@ fn index_metadata_to_segment(metadata: IndexMetadata) -> PyResult<IndexSegment>
     Ok(IndexSegment::new(
         metadata.uuid,
         fragment_bitmap.iter(),
+        metadata.fields,
         index_details,
         metadata.index_version,
+        metadata.dataset_version,
     ))
 }
 
@@ -549,6 +634,23 @@ fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult<Vec<IndexSegm
     Ok(extracted)
 }
 
+fn parse_index_segment_ids(index_segments: Option<Vec<String>>) -> PyResult<Option<Vec<Uuid>>> {
+    index_segments
+        .map(|segments| {
+            segments
+                .into_iter()
+                .map(|segment| {
+                    Uuid::parse_str(&segment).map_err(|err| {
+                        PyValueError::new_err(format!(
+                            "invalid index segment uuid '{segment}': {err}"
+                        ))
+                    })
+                })
+                .collect::<PyResult<Vec<_>>>()
+        })
+        .transpose()
+}
+
 impl MergeInsertBuilder {
     fn build_stats<'a>(stats: &MergeStats, py: Python<'a>) -> PyResult<Bound<'a, PyDict>> {
         let dict = PyDict::new(py);
@@ -1523,18 +1625,21 @@ impl Dataset {
         self_: PyRef<'_, Self>,
         row_ids: Vec<u64>,
         blob_column: &str,
-    ) -> PyResult<Vec<LanceBlobFile>> {
+    ) -> PyResult<Vec<Option<LanceBlobFile>>> {
         let blobs = rt()
             .block_on(Some(self_.py()), self_.ds.take_blobs(&row_ids, blob_column))?
             .infer_error()?;
-        Ok(blobs.into_iter().map(LanceBlobFile::from).collect())
+        Ok(blobs
+            .into_iter()
+            .map(|blob| blob.map(LanceBlobFile::from))
+            .collect())
     }
 
     fn take_blobs_by_addresses(
         self_: PyRef<'_, Self>,
         row_addresses: Vec<u64>,
         blob_column: &str,
-    ) -> PyResult<Vec<LanceBlobFile>> {
+    ) -> PyResult<Vec<Option<LanceBlobFile>>> {
         let blobs = rt()
             .block_on(
                 Some(self_.py()),
@@ -1543,21 +1648,27 @@ impl Dataset {
                     .take_blobs_by_addresses(&row_addresses, blob_column),
             )?
             .infer_error()?;
-        Ok(blobs.into_iter().map(LanceBlobFile::from).collect())
+        Ok(blobs
+            .into_iter()
+            .map(|blob| blob.map(LanceBlobFile::from))
+            .collect())
     }
 
     fn take_blobs_by_indices(
         self_: PyRef<'_, Self>,
         row_indices: Vec<u64>,
         blob_column: &str,
-    ) -> PyResult<Vec<LanceBlobFile>> {
+    ) -> PyResult<Vec<Option<LanceBlobFile>>> {
         let blobs = rt()
             .block_on(
                 Some(self_.py()),
                 self_.ds.take_blobs_by_indices(&row_indices, blob_column),
             )?
             .infer_error()?;
-        Ok(blobs.into_iter().map(LanceBlobFile::from).collect())
+        Ok(blobs
+            .into_iter()
+            .map(|blob| blob.map(LanceBlobFile::from))
+            .collect())
     }
 
     #[pyo3(signature=(
@@ -1572,7 +1683,7 @@ impl Dataset {
         blob_column: &str,
         io_buffer_size: Option<u64>,
         preserve_order: Option<bool>,
-    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+    ) -> PyResult<Vec<PyReadBlob>> {
         let builder = configure_read_blobs_builder(
             self_
                 .ds
@@ -1600,7 +1711,7 @@ impl Dataset {
         blob_column: &str,
         io_buffer_size: Option<u64>,
         preserve_order: Option<bool>,
-    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+    ) -> PyResult<Vec<PyReadBlob>> {
         let builder = configure_read_blobs_builder(
             self_
                 .ds
@@ -1628,7 +1739,7 @@ impl Dataset {
         blob_column: &str,
         io_buffer_size: Option<u64>,
         preserve_order: Option<bool>,
-    ) -> PyResult<Vec<(u64, Py<PyBytes>)>> {
+    ) -> PyResult<Vec<PyReadBlob>> {
         let builder = configure_read_blobs_builder(
             self_
                 .ds
@@ -1644,6 +1755,40 @@ impl Dataset {
         Ok(read_blobs_to_python(self_.py(), blobs))
     }
 
+    #[pyo3(signature=(
+        requests,
+        blob_column,
+        selector,
+        io_buffer_size=None,
+        preserve_order=None
+    ))]
+    fn read_blob_ranges(
+        self_: PyRef<'_, Self>,
+        requests: Vec<(u64, u64, u64)>,
+        blob_column: &str,
+        selector: &str,
+        io_buffer_size: Option<u64>,
+        preserve_order: Option<bool>,
+    ) -> PyResult<Vec<PyReadBlobRange>> {
+        let requests = blob_range_requests_from_tuples(requests);
+        let builder = self_.ds.read_blob_ranges(blob_column).infer_error()?;
+        let builder = match selector {
+            "ids" => builder.with_row_ids(requests),
+            "addresses" => builder.with_row_addresses(requests),
+            "indices" => builder.with_row_indices(requests),
+            selector => {
+                return Err(PyValueError::new_err(format!(
+                    "selector must be one of 'ids', 'addresses', or 'indices', got {selector:?}"
+                )));
+            }
+        };
+        let builder = configure_read_blob_ranges_builder(builder, io_buffer_size, preserve_order);
+        let ranges = rt()
+            .block_on(Some(self_.py()), builder.execute())?
+            .infer_error()?;
+        Ok(read_blob_ranges_to_python(self_.py(), ranges))
+    }
+
     #[pyo3(signature = (row_slices, columns = None, batch_readahead = 10))]
     fn take_scan(
         &self,
@@ -2404,19 +2549,106 @@ impl Dataset {
             "INVERTED" | "FTS" => {
                 let mut params = InvertedIndexParams::default();
                 if let Some(kwargs) = kwargs {
+                    let allowed_kwargs = [
+                        "analyzer",
+                        "with_position",
+                        "base_tokenizer",
+                        "language",
+                        "max_token_length",
+                        "lower_case",
+                        "stem",
+                        "remove_stop_words",
+                        "custom_stop_words",
+                        "ascii_folding",
+                        "min_ngram_length",
+                        "max_ngram_length",
+                        "prefix_only",
+                        "block_size",
+                        "split_identifiers",
+                        "split_on_numerics",
+                        "preserve_original",
+                        "index_operators",
+                        "memory_limit",
+                        "num_workers",
+                        "format_version",
+                        "fragment_ids",
+                        "index_uuid",
+                        "progress_callback",
+                    ];
+                    for (key, _) in kwargs.iter() {
+                        let key: String = key.extract()?;
+                        if !allowed_kwargs.contains(&key.as_str()) {
+                            return Err(PyValueError::new_err(format!(
+                                "unknown FTS index parameter '{}'",
+                                key
+                            )));
+                        }
+                    }
+
+                    let analyzer: Option<String> = kwargs
+                        .get_item("analyzer")?
+                        .map(|value| value.extract())
+                        .transpose()?;
+                    let base_tokenizer: Option<String> = kwargs
+                        .get_item("base_tokenizer")?
+                        .map(|value| value.extract())
+                        .transpose()?;
+
+                    match (analyzer.as_deref(), base_tokenizer.as_deref()) {
+                        (Some("text"), Some("code")) => {
+                            return Err(PyValueError::new_err(
+                                "base_tokenizer='code' requires analyzer='code'",
+                            ));
+                        }
+                        (Some("code"), Some(base_tokenizer)) if base_tokenizer != "code" => {
+                            return Err(PyValueError::new_err(format!(
+                                "analyzer='code' requires base_tokenizer='code', got '{}'",
+                                base_tokenizer
+                            )));
+                        }
+                        _ => {}
+                    }
+
+                    let uses_code_analyzer = match analyzer.as_deref() {
+                        Some("code") => true,
+                        Some("text") | None => base_tokenizer.as_deref() == Some("code"),
+                        Some(_) => true,
+                    };
+                    if !uses_code_analyzer {
+                        for flag in [
+                            "split_identifiers",
+                            "split_on_numerics",
+                            "preserve_original",
+                            "index_operators",
+                        ] {
+                            if let Some(value) = kwargs.get_item(flag)?
+                                && value.extract::<bool>()?
+                            {
+                                return Err(PyValueError::new_err(
+                                    "code analyzer flags require analyzer='code'",
+                                ));
+                            }
+                        }
+                    }
+
+                    if let Some(analyzer) = analyzer {
+                        params = params
+                            .analyzer(&analyzer)
+                            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+                    }
                     if let Some(with_position) = kwargs.get_item("with_position")? {
                         params = params.with_position(with_position.extract()?);
                     }
-                    if let Some(base_tokenizer) = kwargs.get_item("base_tokenizer")? {
-                        params = params.base_tokenizer(base_tokenizer.extract()?);
+                    if let Some(base_tokenizer) = base_tokenizer {
+                        params = params.base_tokenizer(base_tokenizer);
                     }
                     if let Some(language) = kwargs.get_item("language")? {
                         let language: PyBackedStr =
                             language.cast::<PyString>()?.clone().try_into()?;
-                        params = params.language(&language).map_err(|e| {
+                        params = params.language(&language).map_err(|err| {
                             PyValueError::new_err(format!(
-                                "can't set tokenizer language to {}: {:?}",
-                                language, e
+                                "can't set tokenizer language to {}: {}",
+                                language, err
                             ))
                         })?;
                     }
@@ -2432,8 +2664,8 @@ impl Dataset {
                     if let Some(remove_stop_words) = kwargs.get_item("remove_stop_words")? {
                         params = params.remove_stop_words(remove_stop_words.extract()?);
                     }
-                    if let Some(stop_words_file) = kwargs.get_item("custom_stop_words")? {
-                        params = params.custom_stop_words(stop_words_file.extract()?);
+                    if let Some(custom_stop_words) = kwargs.get_item("custom_stop_words")? {
+                        params = params.custom_stop_words(custom_stop_words.extract()?);
                     }
                     if let Some(ascii_folding) = kwargs.get_item("ascii_folding")? {
                         params = params.ascii_folding(ascii_folding.extract()?);
@@ -2447,6 +2679,23 @@ impl Dataset {
                     if let Some(prefix_only) = kwargs.get_item("prefix_only")? {
                         params = params.ngram_prefix_only(prefix_only.extract()?);
                     }
+                    if let Some(block_size) = kwargs.get_item("block_size")? {
+                        params = params
+                            .block_size(block_size.extract()?)
+                            .map_err(|e| PyValueError::new_err(e.to_string()))?;
+                    }
+                    if let Some(split_identifiers) = kwargs.get_item("split_identifiers")? {
+                        params = params.split_identifiers(split_identifiers.extract()?);
+                    }
+                    if let Some(split_on_numerics) = kwargs.get_item("split_on_numerics")? {
+                        params = params.split_on_numerics(split_on_numerics.extract()?);
+                    }
+                    if let Some(preserve_original) = kwargs.get_item("preserve_original")? {
+                        params = params.preserve_original(preserve_original.extract()?);
+                    }
+                    if let Some(index_operators) = kwargs.get_item("index_operators")? {
+                        params = params.index_operators(index_operators.extract()?);
+                    }
                     if let Some(memory_limit) = kwargs.get_item("memory_limit")? {
                         params = params.memory_limit_mb(memory_limit.extract()?);
                     }
@@ -2462,7 +2711,7 @@ impl Dataset {
                             value.to_string()
                         } else {
                             return Err(PyValueError::new_err(
-                                "format_version must be 1, 2, 'v1', or 'v2'",
+                                "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'",
                             ));
                         };
                         let format_version = value
@@ -2600,18 +2849,31 @@ impl Dataset {
         Ok(())
     }
 
-    #[pyo3(signature = (name, *, with_position = false))]
-    fn prewarm_index(&self, name: &str, with_position: bool) -> PyResult<()> {
+    #[pyo3(signature = (name, *, with_position = false, index_segments = None))]
+    fn prewarm_index(
+        &self,
+        name: &str,
+        with_position: bool,
+        index_segments: Option<Vec<String>>,
+    ) -> PyResult<()> {
+        let index_segments = parse_index_segment_ids(index_segments)?;
+
         rt().block_on(None, async {
             if with_position {
-                self.ds
-                    .prewarm_index_with_options(
-                        name,
-                        &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)),
-                    )
-                    .await
+                let options = PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true));
+                if let Some(index_segments) = index_segments.as_deref() {
+                    self.ds
+                        .prewarm_index_segments_with_options(name, index_segments, &options)
+                        .await
+                } else {
+                    self.ds.prewarm_index_with_options(name, &options).await
+                }
             } else {
-                self.ds.prewarm_index(name).await
+                if let Some(index_segments) = index_segments.as_deref() {
+                    self.ds.prewarm_index_segments(name, index_segments).await
+                } else {
+                    self.ds.prewarm_index(name).await
+                }
             }
         })?
         .infer_error()
@@ -3379,7 +3641,6 @@ impl Dataset {
         identity_column=None,
         unsharded=false,
         durable_write=None,
-        sync_indexed_write=None,
         max_wal_buffer_size=None,
         max_wal_flush_interval_ms=None,
         max_memtable_size=None,
@@ -3387,8 +3648,6 @@ impl Dataset {
         max_memtable_batches=None,
         max_unflushed_memtable_bytes=None,
         manifest_scan_batch_size=None,
-        async_index_buffer_rows=None,
-        async_index_interval_ms=None,
         backpressure_log_interval_ms=None,
         stats_log_interval_ms=None,
         hnsw_params=None,
@@ -3402,7 +3661,6 @@ impl Dataset {
         identity_column: Option<String>,
         unsharded: bool,
         durable_write: Option<bool>,
-        sync_indexed_write: Option<bool>,
         max_wal_buffer_size: Option<usize>,
         max_wal_flush_interval_ms: Option<u64>,
         max_memtable_size: Option<usize>,
@@ -3410,8 +3668,6 @@ impl Dataset {
         max_memtable_batches: Option<usize>,
         max_unflushed_memtable_bytes: Option<usize>,
         manifest_scan_batch_size: Option<usize>,
-        async_index_buffer_rows: Option<usize>,
-        async_index_interval_ms: Option<u64>,
         backpressure_log_interval_ms: Option<u64>,
         stats_log_interval_ms: Option<u64>,
         hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -3439,7 +3695,6 @@ impl Dataset {
 
         let writer_config = writer_config_from_kwargs(
             durable_write,
-            sync_indexed_write,
             max_wal_buffer_size,
             max_wal_flush_interval_ms,
             max_memtable_size,
@@ -3447,8 +3702,6 @@ impl Dataset {
             max_memtable_batches,
             max_unflushed_memtable_bytes,
             manifest_scan_batch_size,
-            async_index_buffer_rows,
-            async_index_interval_ms,
             backpressure_log_interval_ms,
             stats_log_interval_ms,
             hnsw_params,
@@ -3530,7 +3783,6 @@ impl Dataset {
         shard_id,
         *,
         durable_write=None,
-        sync_indexed_write=None,
         max_wal_buffer_size=None,
         max_wal_flush_interval_ms=None,
         max_memtable_size=None,
@@ -3538,8 +3790,6 @@ impl Dataset {
         max_memtable_batches=None,
         max_unflushed_memtable_bytes=None,
         manifest_scan_batch_size=None,
-        async_index_buffer_rows=None,
-        async_index_interval_ms=None,
         backpressure_log_interval_ms=None,
         stats_log_interval_ms=None,
         hnsw_params=None,
@@ -3549,7 +3799,6 @@ impl Dataset {
         py: Python<'_>,
         shard_id: String,
         durable_write: Option<bool>,
-        sync_indexed_write: Option<bool>,
         max_wal_buffer_size: Option<usize>,
         max_wal_flush_interval_ms: Option<u64>,
         max_memtable_size: Option<usize>,
@@ -3557,8 +3806,6 @@ impl Dataset {
         max_memtable_batches: Option<usize>,
         max_unflushed_memtable_bytes: Option<usize>,
         manifest_scan_batch_size: Option<usize>,
-        async_index_buffer_rows: Option<usize>,
-        async_index_interval_ms: Option<u64>,
         backpressure_log_interval_ms: Option<u64>,
         stats_log_interval_ms: Option<u64>,
         hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -3570,7 +3817,6 @@ impl Dataset {
 
         let config = writer_config_from_kwargs(
             durable_write,
-            sync_indexed_write,
             max_wal_buffer_size,
             max_wal_flush_interval_ms,
             max_memtable_size,
@@ -3578,8 +3824,6 @@ impl Dataset {
             max_memtable_batches,
             max_unflushed_memtable_bytes,
             manifest_scan_batch_size,
-            async_index_buffer_rows,
-            async_index_interval_ms,
             backpressure_log_interval_ms,
             stats_log_interval_ms,
             hnsw_params,
@@ -3603,9 +3847,10 @@ impl Dataset {
 
     /// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index.
     ///
-    /// This function loads a specific partition from an IVF_FLAT index on a hash column,
-    /// computes pairwise hamming distances between all hashes in the partition,
-    /// filters by threshold, and clusters the results using union-find.
+    /// This function loads a specific partition from every segment of an IVF_FLAT
+    /// index on a hash column, computes pairwise hamming distances between all
+    /// hashes in the combined partition, filters by threshold, and clusters the
+    /// results using union-find.
     ///
     /// Parameters
     /// ----------
@@ -3615,6 +3860,9 @@ impl Dataset {
     ///     The partition ID within the IVF_FLAT index
     /// hamming_threshold : int
     ///     Maximum hamming distance to consider as similar
+    /// index_segments : list of str, optional
+    ///     If specified, only these physical index segment UUIDs of the named
+    ///     logical index contribute rows. Defaults to all segments.
     ///
     /// Returns
     /// -------
@@ -3622,27 +3870,45 @@ impl Dataset {
     ///     A reader yielding batches with columns:
     ///     - 'representative': uint64 - The representative row ID for each cluster
     ///     - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
-    #[pyo3(signature = (index_name, partition_id, hamming_threshold))]
+    #[pyo3(signature = (index_name, partition_id, hamming_threshold, index_segments=None))]
     fn hamming_clustering_for_ivf_partition(
         &self,
         py: Python<'_>,
         index_name: &str,
         partition_id: usize,
         hamming_threshold: u32,
+        index_segments: Option<Vec<String>>,
     ) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
-        use lance::index::vector::hamming::hamming_clustering_for_ivf_partition;
+        use lance::index::vector::hamming::{
+            hamming_clustering_for_ivf_partition, hamming_clustering_for_ivf_partition_segments,
+        };
 
+        let segment_ids = parse_index_segment_ids(index_segments)?;
         let ds = self.ds.as_ref();
         let reader = rt()
-            .block_on(
-                Some(py),
-                hamming_clustering_for_ivf_partition(
-                    ds,
-                    index_name,
-                    partition_id,
-                    hamming_threshold,
-                ),
-            )?
+            .block_on(Some(py), async {
+                match segment_ids.as_deref() {
+                    Some(segment_ids) => {
+                        hamming_clustering_for_ivf_partition_segments(
+                            ds,
+                            index_name,
+                            segment_ids,
+                            partition_id,
+                            hamming_threshold,
+                        )
+                        .await
+                    }
+                    None => {
+                        hamming_clustering_for_ivf_partition(
+                            ds,
+                            index_name,
+                            partition_id,
+                            hamming_threshold,
+                        )
+                        .await
+                    }
+                }
+            })?
             .map_err(|err| PyValueError::new_err(err.to_string()))?;
 
         Ok(PyArrowType(reader))
@@ -3650,26 +3916,43 @@ impl Dataset {
 
     /// Get partition information for an IVF_FLAT index.
     ///
+    /// Partition sizes are aggregated across all segments of the logical index
+    /// unless a subset is selected via ``index_segments``.
+    ///
     /// Parameters
     /// ----------
     /// index_name : str
     ///     Name of the IVF_FLAT index
+    /// index_segments : list of str, optional
+    ///     If specified, only these physical index segment UUIDs of the named
+    ///     logical index contribute to the sizes. Defaults to all segments.
     ///
     /// Returns
     /// -------
     /// List[dict]
     ///     List of partition info dicts with 'partition_id' and 'size'
-    #[pyo3(signature = (index_name))]
+    #[pyo3(signature = (index_name, index_segments=None))]
     fn get_ivf_partition_info(
         &self,
         py: Python<'_>,
         index_name: &str,
+        index_segments: Option<Vec<String>>,
     ) -> PyResult<Vec<Py<PyDict>>> {
-        use lance::index::vector::hamming::get_ivf_partition_info;
+        use lance::index::vector::hamming::{
+            get_ivf_partition_info, get_ivf_partition_info_segments,
+        };
 
+        let segment_ids = parse_index_segment_ids(index_segments)?;
         let ds = self.ds.as_ref();
         let result = rt()
-            .block_on(Some(py), get_ivf_partition_info(ds, index_name))?
+            .block_on(Some(py), async {
+                match segment_ids.as_deref() {
+                    Some(segment_ids) => {
+                        get_ivf_partition_info_segments(ds, index_name, segment_ids).await
+                    }
+                    None => get_ivf_partition_info(ds, index_name).await,
+                }
+            })?
             .map_err(|err| PyValueError::new_err(err.to_string()))?;
 
         let partitions: PyResult<Vec<_>> = result
@@ -3694,7 +3977,8 @@ impl Dataset {
     /// Parameters
     /// ----------
     /// column : str
-    ///     Name of the hash column (must be FixedSizeList<UInt8, 8>)
+    ///     Name of the hash column (must be FixedSizeList<UInt8, N> where N is
+    ///     a positive multiple of 8 bytes)
     /// sample_size : int, optional
     ///     Number of rows to sample (if None or >= total rows, uses all rows)
     /// hamming_threshold : int
@@ -3737,7 +4021,8 @@ impl Dataset {
     /// Parameters
     /// ----------
     /// column : str
-    ///     Name of the hash column (must be FixedSizeList<UInt8, 8>)
+    ///     Name of the hash column (must be FixedSizeList<UInt8, N> where N is
+    ///     a positive multiple of 8 bytes)
     /// fragment_id : int
     ///     The fragment ID to read from
     /// start_row : int
diff --git a/python/src/executor.rs b/python/src/executor.rs
index e2e573a8193..e2c1ca4b383 100644
--- a/python/src/executor.rs
+++ b/python/src/executor.rs
@@ -19,6 +19,10 @@ use pyo3::{PyResult, Python, exceptions::PyRuntimeError};
 
 pub const SIGNAL_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100);
 
+fn is_python314_or_later(py: Option<Python<'_>>) -> bool {
+    py.is_some_and(|py| py.version_info() >= (3, 14))
+}
+
 /// A wrapper around tokio runtime.
 ///
 /// This is used to spawn tasks in the background and wait synchronously for them
@@ -195,6 +199,8 @@ impl BackgroundExecutor {
         F::Output: Send,
         P: FnMut() -> PyResult<()>,
     {
+        let should_propagate_on_completion = is_python314_or_later(py);
+
         let mut future = std::pin::pin!(future);
 
         loop {
@@ -236,11 +242,22 @@ impl BackgroundExecutor {
             };
 
             if let Some(output) = maybe_output {
-                if let Err(err) = pump() {
-                    log::warn!(
-                        "Ignoring progress callback error after operation completed successfully: {}",
-                        err
-                    );
+                // When the index build finishes so fast that no pump cycles
+                // occurred during execution, pending events sit in the channel
+                // buffer and get drained after completion. Python ≥ 3.14 changed
+                // GIL/async scheduling timing such that callback invocations may
+                // only be visible in this post-completion drain, so we propagate
+                // errors for 3.14+. For ≤ 3.13 we keep the old tolerant behavior
+                // to avoid spurious failures when scheduling shifts slightly.
+                if should_propagate_on_completion {
+                    pump()?;
+                } else {
+                    if let Err(err) = pump() {
+                        log::warn!(
+                            "Ignoring progress callback error after operation completed successfully: {}",
+                            err
+                        );
+                    }
                 }
                 return Ok(output);
             }
diff --git a/python/src/fragment.rs b/python/src/fragment.rs
index dbe5c426903..6b832870280 100644
--- a/python/src/fragment.rs
+++ b/python/src/fragment.rs
@@ -26,6 +26,7 @@ use lance::dataset::transaction::{Operation, Transaction};
 use lance::dataset::{InsertBuilder, NewColumnTransform, WriteParams};
 use lance_core::datatypes::BlobHandling;
 use lance_io::utils::CachedFileSize;
+use lance_table::format::overlay::DataOverlayFile;
 use lance_table::format::{
     DataFile, DeletionFile, DeletionFileType, Fragment, RowDatasetVersionMeta, RowIdMeta,
 };
@@ -825,6 +826,10 @@ impl FromPyObject<'_, '_> for PyLance<Fragment> {
             row_id_meta,
             last_updated_at_version_meta,
             created_at_version_meta,
+            // Round-tripped so overlays survive operations that pass existing
+            // fragments back (a manual Delete/Update/Merge commit). Sorting
+            // newest-last is deferred to the manifest reload after commit.
+            overlays: extract_vec::<DataOverlayFile>(&ob.getattr("overlays")?)?,
         }))
     }
 }
@@ -857,6 +862,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> {
             .created_at_version_meta
             .as_ref()
             .map(|r| PyRowDatasetVersionMeta(r.clone()));
+        let overlays = export_vec(py, &self.0.overlays)?;
 
         cls.call1((
             self.0.id,
@@ -866,6 +872,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> {
             row_id_meta,
             created_at_version_meta,
             last_updated_at_version_meta,
+            overlays,
         ))
     }
 }
diff --git a/python/src/indices.rs b/python/src/indices.rs
index 7ce7a297924..cdf52b37fa8 100644
--- a/python/src/indices.rs
+++ b/python/src/indices.rs
@@ -232,6 +232,7 @@ async fn do_train_pq_model(
     distance_type: &str,
     sample_rate: u32,
     max_iters: u32,
+    num_bits: u32,
     ivf_model: IvfModel,
     fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<ArrayData> {
@@ -239,7 +240,7 @@ async fn do_train_pq_model(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let params = PQBuildParams {
         num_sub_vectors: num_subvectors as usize,
-        num_bits: 8,
+        num_bits: num_bits as usize,
         max_iters: max_iters as usize,
         sample_rate: sample_rate as usize,
         ..Default::default()
@@ -260,7 +261,7 @@ async fn do_train_pq_model(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
-#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None))]
+#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None, num_bits=8))]
 fn train_pq_model<'py>(
     py: Python<'py>,
     dataset: &Dataset,
@@ -272,6 +273,7 @@ fn train_pq_model<'py>(
     max_iters: u32,
     ivf_centroids: PyArrowType<ArrayData>,
     fragment_ids: Option<Vec<u32>>,
+    num_bits: u32,
 ) -> PyResult<Bound<'py, PyAny>> {
     let ivf_centroids = ivf_centroids.0;
     let ivf_centroids = FixedSizeListArray::from(ivf_centroids);
@@ -291,6 +293,7 @@ fn train_pq_model<'py>(
             distance_type,
             sample_rate,
             max_iters,
+            num_bits,
             ivf_model,
             fragment_ids,
         ),
@@ -398,7 +401,7 @@ async fn do_transform_vectors(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
-#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None))]
+#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None, num_bits=8))]
 pub fn transform_vectors(
     py: Python<'_>,
     dataset: &Dataset,
@@ -411,6 +414,7 @@ pub fn transform_vectors(
     dst_uri: &str,
     fragments: Vec<FileFragment>,
     partitions_ds_uri: Option<&str>,
+    num_bits: u32,
 ) -> PyResult<()> {
     let ivf_centroids = ivf_centroids.0;
     let ivf_centroids = FixedSizeListArray::from(ivf_centroids);
@@ -419,7 +423,7 @@ pub fn transform_vectors(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let pq = ProductQuantizer::new(
         num_subvectors as usize,
-        /*num_bits=*/ 8,
+        num_bits,
         dimension,
         codebook,
         distance_type,
@@ -546,12 +550,14 @@ async fn do_load_shuffled_vectors(
             .as_ref()
             .expect("vector metadata should include fragment coverage")
             .iter(),
+        metadata.fields.iter().copied(),
         metadata
             .index_details
             .as_ref()
             .expect("vector metadata should include index details")
             .clone(),
         metadata.index_version,
+        metadata.dataset_version,
     );
     ds.commit_existing_index_segments(index_name, column, vec![segment])
         .await
@@ -561,7 +567,7 @@ async fn do_load_shuffled_vectors(
 }
 
 #[pyfunction]
-#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None))]
+#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None, num_bits=8))]
 #[allow(clippy::too_many_arguments)]
 pub fn load_shuffled_vectors(
     filenames: Vec<String>,
@@ -574,6 +580,7 @@ pub fn load_shuffled_vectors(
     num_subvectors: u32,
     distance_type: &str,
     index_name: Option<&str>,
+    num_bits: u32,
 ) -> PyResult<()> {
     let mut default_idx_name = column.to_string();
     default_idx_name.push_str("_idx");
@@ -595,7 +602,7 @@ pub fn load_shuffled_vectors(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let pq_model = ProductQuantizer::new(
         num_subvectors as usize,
-        /*num_bits=*/ 8,
+        num_bits,
         pq_dimension,
         codebook,
         distance_type,
diff --git a/python/src/lib.rs b/python/src/lib.rs
index 466d4ea90f2..f8afa704c10 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -74,6 +74,7 @@ pub(crate) mod fragment;
 pub(crate) mod indices;
 pub(crate) mod mem_wal;
 pub(crate) mod namespace;
+pub(crate) mod otel;
 pub(crate) mod reader;
 pub(crate) mod scanner;
 pub(crate) mod schema;
@@ -298,7 +299,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<namespace::PyRestAdapter>()?;
     m.add_class::<storage_options::PyStorageOptionsAccessor>()?;
     // MemWAL classes
-    m.add_class::<mem_wal::PyMergedGeneration>()?;
+    m.add_class::<mem_wal::PyCompactedSsTable>()?;
     m.add_class::<mem_wal::PyShardSnapshot>()?;
     m.add_class::<mem_wal::PyShardWriter>()?;
     m.add_class::<mem_wal::PyLsmScanner>()?;
@@ -318,10 +319,17 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(trace_to_chrome))?;
     m.add_wrapped(wrap_pyfunction!(capture_trace_events))?;
     m.add_wrapped(wrap_pyfunction!(shutdown_tracing))?;
+    // OpenTelemetry metrics bridge
+    m.add_class::<otel::PyMetricPoint>()?;
+    m.add_class::<otel::PyMetricDescription>()?;
+    m.add_wrapped(wrap_pyfunction!(otel::register_lance_metrics_recorder))?;
+    m.add_wrapped(wrap_pyfunction!(otel::lance_metrics_catalog))?;
+    m.add_wrapped(wrap_pyfunction!(otel::snapshot_lance_metrics))?;
     m.add_wrapped(wrap_pyfunction!(manifest_needs_migration))?;
     m.add_wrapped(wrap_pyfunction!(language_model_home))?;
     m.add_wrapped(wrap_pyfunction!(bytes_read_counter))?;
     m.add_wrapped(wrap_pyfunction!(iops_counter))?;
+    m.add_wrapped(wrap_pyfunction!(simd_info))?;
     m.add_wrapped(wrap_pyfunction!(stable_version))?;
     // Debug functions
     m.add_wrapped(wrap_pyfunction!(debug::format_schema))?;
@@ -340,6 +348,37 @@ fn iops_counter() -> PyResult<u64> {
     Ok(::lance::io::iops_counter())
 }
 
+/// Returns a dict describing which SIMD tier the lance runtime dispatches to
+/// on this host, plus the raw CPU feature flags it detected.
+///
+/// Mirrors `pyarrow.runtime_info()`: a cheap, transparent way to verify that
+/// the host is hitting the expected SIMD tier (e.g., `"avx512_fp16"`,
+/// `"avx2"`) when debugging vector-search performance.
+///
+/// Returns:
+///   {
+///     "tier": str,                      # e.g. "avx2", "avx_fma", "neon", "none"
+///     "target_arch": str,               # e.g. "x86_64", "aarch64", "loongarch64"
+///     "host_features": list[str],       # raw CPU feature flags (x86_64 only)
+///   }
+///
+/// Examples:
+///   >>> import lance
+///   >>> info = lance.simd_info()
+///   >>> sorted(info)
+///   ['host_features', 'target_arch', 'tier']
+///   >>> isinstance(info["tier"], str)
+///   True
+#[pyfunction]
+pub fn simd_info(py: Python<'_>) -> PyResult<Py<PyAny>> {
+    let info = lance_core::utils::cpu::simd_info();
+    let dict = pyo3::types::PyDict::new(py);
+    dict.set_item("tier", info.tier.to_string())?;
+    dict.set_item("target_arch", info.target_arch)?;
+    dict.set_item("host_features", info.host_features)?;
+    Ok(dict.into())
+}
+
 #[pyfunction(name = "bytes_read_counter")]
 fn bytes_read_counter() -> PyResult<u64> {
     Ok(::lance::io::bytes_read_counter())
diff --git a/python/src/mem_wal.rs b/python/src/mem_wal.rs
index 812c5f42f22..2d6f2487a75 100644
--- a/python/src/mem_wal.rs
+++ b/python/src/mem_wal.rs
@@ -18,13 +18,13 @@ use datafusion::prelude::SessionContext;
 use futures::TryStreamExt;
 use lance::dataset::Dataset as LanceDataset;
 use lance::dataset::mem_wal::scanner::{
-    FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner,
+    LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, SsTable,
     parse_filter_expr as parse_lsm_filter_expr,
 };
 use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot};
 use lance::dataset::mem_wal::{LsmScanner, ShardSnapshot, ShardWriter, evaluate_sharding_spec};
 use lance_index::mem_wal::{
-    MergedGeneration as LanceMergedGeneration, ShardingField, ShardingSpec,
+    CompactedSsTable as LanceCompactedSsTable, ShardingField, ShardingSpec,
 };
 use lance_linalg::distance::DistanceType;
 use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError};
@@ -52,10 +52,10 @@ pub fn py_evaluate_sharding_spec<'py>(
     result.to_pyarrow(py)
 }
 
-/// Write a primary-key dedup sidecar (`_pk_index/`) for a flushed-generation
+/// Write a primary-key dedup sidecar (`_pk_index/`) for an SSTable
 /// dataset already written at `gen_path`, mirroring what production flush emits.
 ///
-/// Test-support only: lets Python tests stage a *faithful* flushed generation
+/// Test-support only: lets Python tests stage a *faithful* SSTable
 /// (dataset + sidecar). Production always writes the sidecar during flush, so a
 /// dataset-without-sidecar is not a state the system otherwise produces.
 #[pyfunction(name = "_write_pk_sidecar", signature = (gen_path, data, pk_columns))]
@@ -115,16 +115,17 @@ fn optional_string(value: Bound<'_, PyAny>) -> PyResult<Option<String>> {
     }
 }
 
-/// Represents a single generation of a MemWAL shard that has been merged
-/// into the base table. Used with `MergeInsertBuilder.mark_generations_as_merged()`.
-#[pyclass(name = "_MergedGeneration", module = "_lib")]
-pub struct PyMergedGeneration {
+/// Points to an SSTable compacted into the base table.
+///
+/// Used with `MergeInsertBuilder.mark_sstables_as_compacted()`.
+#[pyclass(name = "_CompactedSsTable", module = "_lib")]
+pub struct PyCompactedSsTable {
     pub shard_id: String,
     pub generation: u64,
 }
 
 #[pymethods]
-impl PyMergedGeneration {
+impl PyCompactedSsTable {
     #[new]
     pub fn new(shard_id: String, generation: u64) -> Self {
         Self {
@@ -145,23 +146,23 @@ impl PyMergedGeneration {
 
     pub fn __repr__(&self) -> String {
         format!(
-            "_MergedGeneration(shard_id='{}', generation={})",
+            "_CompactedSsTable(shard_id='{}', generation={})",
             self.shard_id, self.generation
         )
     }
 }
 
-impl PyMergedGeneration {
-    pub fn to_lance(&self) -> PyResult<LanceMergedGeneration> {
+impl PyCompactedSsTable {
+    pub fn to_lance(&self) -> PyResult<LanceCompactedSsTable> {
         let uuid = Uuid::parse_str(&self.shard_id)
             .map_err(|e| PyValueError::new_err(format!("Invalid shard_id UUID: {}", e)))?;
-        Ok(LanceMergedGeneration::new(uuid, self.generation))
+        Ok(LanceCompactedSsTable::new(uuid, self.generation))
     }
 }
 
 /// Snapshot of a MemWAL shard's state at a point in time.
 ///
-/// Used to specify which flushed generations to include when creating an
+/// Used to specify which SSTables to include when creating an
 /// `_LsmScanner`. Supports a builder pattern for adding generations.
 #[pyclass(name = "_ShardSnapshot", module = "_lib", skip_from_py_object)]
 #[derive(Clone)]
@@ -195,13 +196,13 @@ impl PyShardSnapshot {
         slf
     }
 
-    /// Add a flushed generation by its generation number and storage path.
-    pub fn with_flushed_generation(
+    /// Add an SSTable by its generation number and storage path.
+    pub fn with_sstable(
         mut slf: PyRefMut<'_, Self>,
         generation: u64,
         path: String,
     ) -> PyRefMut<'_, Self> {
-        slf.inner = slf.inner.clone().with_flushed_generation(generation, path);
+        slf.inner = slf.inner.clone().with_sstable(generation, path);
         slf
     }
 
@@ -212,10 +213,10 @@ impl PyShardSnapshot {
 
     pub fn __repr__(&self) -> String {
         format!(
-            "_ShardSnapshot(shard_id='{}', current_gen={}, flushed_gens={})",
+            "_ShardSnapshot(shard_id='{}', current_gen={}, sstables={})",
             self.inner.shard_id,
             self.inner.current_generation,
-            self.inner.flushed_generations.len()
+            self.inner.sstables.len()
         )
     }
 }
@@ -238,6 +239,14 @@ struct ClosedShardWriterState {
     memtable_stats: MemTableStats,
 }
 
+fn collect_record_batches(data: &Bound<'_, PyAny>) -> PyResult<Vec<RecordBatch>> {
+    let reader = ArrowArrayStreamReader::from_pyarrow_bound(data)
+        .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?;
+    reader
+        .collect::<Result<_, _>>()
+        .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))
+}
+
 #[pymethods]
 impl PyShardWriter {
     /// Write data batches to the MemWAL.
@@ -245,11 +254,7 @@ impl PyShardWriter {
     /// Accepts any PyArrow-compatible data source (RecordBatch, Table,
     /// or an Arrow stream reader).
     pub fn put(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> {
-        let reader = ArrowArrayStreamReader::from_pyarrow_bound(data)
-            .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?;
-        let batches: Vec<RecordBatch> = reader
-            .collect::<Result<_, _>>()
-            .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))?;
+        let batches = collect_record_batches(data)?;
 
         if batches.is_empty() {
             return Ok(());
@@ -268,6 +273,31 @@ impl PyShardWriter {
         .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))
     }
 
+    /// Delete rows from the MemWAL by primary key.
+    ///
+    /// Accepts any PyArrow-compatible data source carrying the shard's primary
+    /// key column(s). Rust core validates that primary keys exist and builds the
+    /// tombstone rows.
+    pub fn delete(&self, py: Python<'_>, keys: &Bound<'_, PyAny>) -> PyResult<()> {
+        let batches = collect_record_batches(keys)?;
+
+        if batches.is_empty() {
+            return Ok(());
+        }
+
+        let inner = self.inner.clone();
+        rt().block_on(Some(py), async move {
+            let guard = inner.lock().await;
+            match guard.as_ref() {
+                Some(writer) => writer.delete(batches).await.map(|_| ()),
+                None => Err(lance_core::Error::invalid_input(
+                    "ShardWriter is already closed",
+                )),
+            }
+        })?
+        .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))
+    }
+
     /// Flush pending data and close the writer.
     ///
     /// After close(), calling put() will raise an error.
@@ -355,7 +385,7 @@ impl PyShardWriter {
 
     /// Create an LSM scanner that includes the active MemTable for strong consistency.
     ///
-    /// The scanner covers: base table + given flushed generations + current active MemTable.
+    /// The scanner covers: base table + given SSTables + current active MemTable.
     #[pyo3(signature = (shard_snapshots=vec![]))]
     pub fn lsm_scanner(
         &self,
@@ -508,7 +538,7 @@ impl PyExecutionPlan {
     }
 }
 
-/// LSM-aware scanner covering base table, flushed MemTables, and active MemTable.
+/// LSM-aware scanner covering base table, SSTables, and active MemTable.
 ///
 /// Provides deduplication by primary key, always returning the newest version
 /// of each row across all LSM levels.
@@ -920,10 +950,8 @@ fn memtable_stats_to_pydict(py: Python<'_>, stats: &MemTableStats) -> PyResult<P
         "max_buffered_batch_position",
         stats.max_buffered_batch_position,
     )?;
-    dict.set_item(
-        "max_flushed_batch_position",
-        stats.max_flushed_batch_position,
-    )?;
+    dict.set_item("durable_batch_count", stats.durable_batch_count)?;
+    dict.set_item("global_offset", stats.global_offset)?;
     dict.set_item(
         "pending_wal_start_batch_position",
         stats.pending_wal_start_batch_position,
@@ -994,12 +1022,12 @@ fn shard_snapshot_from_manifest(manifest: lance_index::mem_wal::ShardManifest) -
         shard_id: manifest.shard_id,
         spec_id: manifest.shard_spec_id,
         current_generation: manifest.current_generation,
-        flushed_generations: manifest
-            .flushed_generations
+        sstables: manifest
+            .sstables
             .into_iter()
-            .map(|generation| FlushedGeneration {
-                generation: generation.generation,
-                path: generation.path,
+            .map(|sstable| SsTable {
+                generation: sstable.generation,
+                path: sstable.path,
             })
             .collect(),
     }
@@ -1010,13 +1038,18 @@ fn closed_memtable_stats(stats_before_close: MemTableStats) -> MemTableStats {
         return stats_before_close;
     }
 
+    // After a successful close every buffered batch is flushed and WAL-durable,
+    // so the synthesized empty memtable starts at the writer's global end and the
+    // durable cursor has caught up to it.
+    let global_end = stats_before_close.global_offset + stats_before_close.batch_count;
     MemTableStats {
         row_count: 0,
         batch_count: 0,
         estimated_size: 0,
         generation: stats_before_close.generation.saturating_add(1),
         max_buffered_batch_position: None,
-        max_flushed_batch_position: None,
+        durable_batch_count: global_end,
+        global_offset: global_end,
         pending_wal_start_batch_position: None,
         pending_wal_end_batch_position: None,
         pending_wal_batch_count: 0,
diff --git a/python/src/otel.rs b/python/src/otel.rs
new file mode 100644
index 00000000000..71d191d7db9
--- /dev/null
+++ b/python/src/otel.rs
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Bridge from the [`metrics`] crate facade to Python OpenTelemetry.
+//!
+//! Lance core publishes metrics through the global [`metrics`] facade without
+//! choosing a backend. This module installs a process-global [`Recorder`] that
+//! aggregates those metrics into lock-free cumulative storage, and exposes that
+//! state to Python so the bindings can feed it into the user's OpenTelemetry
+//! `MeterProvider`.
+//!
+//! While it targets OpenTelemetry on the Python side, the recorder is agnostic
+//! to the metric *source*: it records any metric emitted through the facade,
+//! keyed by name and labels. Object store metrics are the first producer, but
+//! nothing here is specific to them. New metrics flow through automatically;
+//! they only need to be described (see [`describe_all`]) so the Python layer can
+//! discover their name, kind, and unit up front.
+//!
+//! ## Why pull, not push
+//!
+//! OpenTelemetry collects on its own schedule and invokes observable-instrument
+//! callbacks at collection time. Cumulative counters map directly onto OTel's
+//! `ObservableCounter` semantics. So the bridge aggregates in Rust and lets the
+//! Python collection thread pull a [`snapshot`](snapshot_lance_metrics). The
+//! snapshot is lock-free, but it still walks every registered series and
+//! allocates owned copies of their names and labels, so it runs with the GIL
+//! released to avoid stalling other Python threads during collection.
+//!
+//! ## Histograms
+//!
+//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot
+//! be pulled as-is. Instead each histogram is aggregated into fixed buckets
+//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count
+//! and sum, which the Python layer surfaces as observable counters.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
+
+use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
+use metrics_util::registry::{Registry, Storage};
+use pyo3::prelude::*;
+
+/// Bucket boundaries used when a histogram has no registered bounds. Covers a
+/// broad latency range so unknown histograms still produce useful buckets.
+const DEFAULT_BOUNDS: &[f64] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
+];
+
+/// The kind of a metric, mirroring the three `metrics` instrument types.
+#[derive(Clone, Copy)]
+enum MetricKind {
+    Counter,
+    Gauge,
+    Histogram,
+}
+
+impl MetricKind {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Counter => "counter",
+            Self::Gauge => "gauge",
+            Self::Histogram => "histogram",
+        }
+    }
+}
+
+/// Description of a metric, populated by the recorder's `describe_*` methods.
+struct MetricDescription {
+    kind: MetricKind,
+    unit: Option<String>,
+    description: String,
+}
+
+/// Catalog of described metrics, keyed by metric name. The Python layer reads
+/// this to create one OpenTelemetry instrument per metric up front.
+static CATALOG: LazyLock<Mutex<HashMap<String, MetricDescription>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Per-metric histogram bucket boundaries, keyed by metric name. Producers
+/// register their recommended bounds before any metric is recorded.
+static HISTOGRAM_BOUNDS: LazyLock<RwLock<HashMap<String, Arc<[f64]>>>> =
+    LazyLock::new(|| RwLock::new(HashMap::new()));
+
+/// The installed recorder's registry, available once installation succeeds.
+static REGISTRY: OnceLock<Arc<Registry<Key, LanceStorage>>> = OnceLock::new();
+
+fn bounds_for(name: &str) -> Arc<[f64]> {
+    HISTOGRAM_BOUNDS
+        .read()
+        .unwrap()
+        .get(name)
+        .cloned()
+        .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS))
+}
+
+/// A histogram that buckets samples at record time into fixed boundaries,
+/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded
+/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets.
+struct BucketedHistogram {
+    /// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose
+    /// bound is `>= v`; samples above all bounds fall in the implicit `+Inf`
+    /// bucket stored as the final entry of `counts`.
+    bounds: Arc<[f64]>,
+    /// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`.
+    counts: Box<[AtomicU64]>,
+    count: AtomicU64,
+    /// Running sum of recorded values, stored as `f64` bits (there is no atomic
+    /// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]).
+    sum_bits: AtomicU64,
+}
+
+// All atomics here use `Ordering::Relaxed`: each metric counter is independent,
+// so no happens-before relationship is needed between them, and a snapshot
+// reader tolerates slightly stale values. This matches `metrics_util`'s
+// `AtomicStorage`.
+
+impl BucketedHistogram {
+    fn new(bounds: Arc<[f64]>) -> Self {
+        let counts = (0..bounds.len() + 1)
+            .map(|_| AtomicU64::new(0))
+            .collect::<Vec<_>>()
+            .into_boxed_slice();
+        Self {
+            bounds,
+            counts,
+            count: AtomicU64::new(0),
+            sum_bits: AtomicU64::new(0),
+        }
+    }
+
+    fn add_to_sum(&self, value: f64) {
+        // No atomic offers an f64 add, so read the current bit pattern, add in
+        // float space, and CAS it back, retrying if another thread won the race.
+        let mut current = self.sum_bits.load(Ordering::Relaxed);
+        loop {
+            let updated = (f64::from_bits(current) + value).to_bits();
+            match self.sum_bits.compare_exchange_weak(
+                current,
+                updated,
+                Ordering::Relaxed,
+                Ordering::Relaxed,
+            ) {
+                Ok(_) => break,
+                Err(actual) => current = actual,
+            }
+        }
+    }
+
+    /// Cumulative `le` buckets, total count, and sum at this instant.
+    fn snapshot(&self) -> MetricValue {
+        let mut cumulative = 0u64;
+        let mut buckets = Vec::with_capacity(self.bounds.len() + 1);
+        for (i, bound) in self.bounds.iter().enumerate() {
+            cumulative += self.counts[i].load(Ordering::Relaxed);
+            buckets.push((format!("{}", bound), cumulative));
+        }
+        cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed);
+        buckets.push(("+Inf".to_string(), cumulative));
+        MetricValue::Histogram {
+            buckets,
+            count: self.count.load(Ordering::Relaxed),
+            sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)),
+        }
+    }
+}
+
+impl metrics::HistogramFn for BucketedHistogram {
+    fn record(&self, value: f64) {
+        let idx = self.bounds.partition_point(|&bound| bound < value);
+        self.counts[idx].fetch_add(1, Ordering::Relaxed);
+        self.count.fetch_add(1, Ordering::Relaxed);
+        self.add_to_sum(value);
+    }
+}
+
+/// Storage backing the registry. Counters and gauges are plain atomics (as in
+/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`].
+struct LanceStorage;
+
+impl Storage<Key> for LanceStorage {
+    type Counter = Arc<AtomicU64>;
+    type Gauge = Arc<AtomicU64>;
+    type Histogram = Arc<BucketedHistogram>;
+
+    fn counter(&self, _key: &Key) -> Self::Counter {
+        Arc::new(AtomicU64::new(0))
+    }
+
+    fn gauge(&self, _key: &Key) -> Self::Gauge {
+        // The `metrics` facade writes the f64 bit pattern into this `u64` (the
+        // snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`.
+        // `0` decodes to `0.0`, the correct initial value.
+        Arc::new(AtomicU64::new(0))
+    }
+
+    fn histogram(&self, key: &Key) -> Self::Histogram {
+        Arc::new(BucketedHistogram::new(bounds_for(key.name())))
+    }
+}
+
+struct LanceRecorder {
+    registry: Arc<Registry<Key, LanceStorage>>,
+}
+
+impl LanceRecorder {
+    fn describe(
+        &self,
+        key: KeyName,
+        kind: MetricKind,
+        unit: Option<Unit>,
+        description: SharedString,
+    ) {
+        CATALOG.lock().unwrap().insert(
+            key.as_str().to_string(),
+            MetricDescription {
+                kind,
+                unit: unit.map(|u| u.as_canonical_label().to_string()),
+                description: description.into_owned(),
+            },
+        );
+    }
+}
+
+impl Recorder for LanceRecorder {
+    fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Counter, unit, description);
+    }
+
+    fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Gauge, unit, description);
+    }
+
+    fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Histogram, unit, description);
+    }
+
+    fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
+        self.registry
+            .get_or_create_counter(key, |c| Counter::from_arc(c.clone()))
+    }
+
+    fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge {
+        self.registry
+            .get_or_create_gauge(key, |g| Gauge::from_arc(g.clone()))
+    }
+
+    fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram {
+        self.registry
+            .get_or_create_histogram(key, |h| Histogram::from_arc(h.clone()))
+    }
+}
+
+/// Register the recommended histogram bounds for every metric-emitting
+/// subsystem. New subsystems add their `histogram_bounds()` here.
+fn register_bounds() {
+    let mut bounds = HISTOGRAM_BOUNDS.write().unwrap();
+    for (name, values) in lance_io::object_store::metrics::histogram_bounds() {
+        bounds.insert((*name).to_string(), Arc::from(*values));
+    }
+}
+
+/// Describe every metric-emitting subsystem so the catalog is populated. Must
+/// run after the recorder is installed. New subsystems add their
+/// `describe_metrics()` here.
+fn describe_all() {
+    lance_io::object_store::metrics::describe_metrics();
+}
+
+enum MetricValue {
+    Scalar(f64),
+    Histogram {
+        buckets: Vec<(String, u64)>,
+        count: u64,
+        sum: f64,
+    },
+}
+
+struct MetricPoint {
+    name: String,
+    kind: &'static str,
+    attributes: HashMap<String, String>,
+    value: MetricValue,
+}
+
+fn labels(key: &Key) -> HashMap<String, String> {
+    key.labels()
+        .map(|label| (label.key().to_string(), label.value().to_string()))
+        .collect()
+}
+
+fn collect_points(registry: &Registry<Key, LanceStorage>) -> Vec<MetricPoint> {
+    let mut points = Vec::new();
+    for (key, handle) in registry.get_counter_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "counter",
+            attributes: labels(&key),
+            // OpenTelemetry observations are float; counts stay well within the
+            // f64-exact integer range (2^53), so this cast is lossless in practice.
+            value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64),
+        });
+    }
+    for (key, handle) in registry.get_gauge_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "gauge",
+            attributes: labels(&key),
+            value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))),
+        });
+    }
+    for (key, handle) in registry.get_histogram_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "histogram",
+            attributes: labels(&key),
+            value: handle.snapshot(),
+        });
+    }
+    points
+}
+
+/// One metric data point exposed to Python. For counters and gauges only
+/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`,
+/// and `sum` are set.
+#[pyclass(name = "MetricPoint", get_all)]
+pub struct PyMetricPoint {
+    name: String,
+    kind: String,
+    attributes: HashMap<String, String>,
+    value: Option<f64>,
+    buckets: Option<Vec<(String, u64)>>,
+    count: Option<u64>,
+    sum: Option<f64>,
+}
+
+impl From<MetricPoint> for PyMetricPoint {
+    fn from(point: MetricPoint) -> Self {
+        let (value, buckets, count, sum) = match point.value {
+            MetricValue::Scalar(v) => (Some(v), None, None, None),
+            MetricValue::Histogram {
+                buckets,
+                count,
+                sum,
+            } => (None, Some(buckets), Some(count), Some(sum)),
+        };
+        Self {
+            name: point.name,
+            kind: point.kind.to_string(),
+            attributes: point.attributes,
+            value,
+            buckets,
+            count,
+            sum,
+        }
+    }
+}
+
+/// A described metric, used by the Python layer to create instruments up front.
+#[pyclass(name = "MetricDescription", get_all)]
+pub struct PyMetricDescription {
+    name: String,
+    kind: String,
+    unit: Option<String>,
+    description: String,
+}
+
+/// Install the Lance metrics recorder as the process-global `metrics` recorder.
+///
+/// Returns `True` if the recorder is installed (now or previously). Returns
+/// `False` if a *different* recorder is already installed — `metrics` allows
+/// only one global recorder per process, so Lance cannot coexist with another.
+#[pyfunction]
+pub fn register_lance_metrics_recorder() -> bool {
+    if REGISTRY.get().is_some() {
+        return true;
+    }
+    let registry = Arc::new(Registry::new(LanceStorage));
+    let recorder = LanceRecorder {
+        registry: registry.clone(),
+    };
+    // Register histogram bounds *before* installing the recorder. Once the
+    // recorder is global, another thread can emit a metric and create the
+    // histogram handle concurrently; if the bounds aren't registered yet that
+    // handle would be built with the fallback bounds and keep them for the
+    // life of the process. `register_bounds()` doesn't need the recorder.
+    register_bounds();
+    match metrics::set_global_recorder(recorder) {
+        Ok(()) => {
+            let _ = REGISTRY.set(registry);
+            describe_all();
+            true
+        }
+        Err(_) => false,
+    }
+}
+
+/// The catalog of described Lance metrics. Empty until the recorder is installed.
+#[pyfunction]
+pub fn lance_metrics_catalog() -> Vec<PyMetricDescription> {
+    CATALOG
+        .lock()
+        .unwrap()
+        .iter()
+        .map(|(name, desc)| PyMetricDescription {
+            name: name.clone(),
+            kind: desc.kind.as_str().to_string(),
+            unit: desc.unit.clone(),
+            description: desc.description.clone(),
+        })
+        .collect()
+}
+
+/// A point-in-time snapshot of every recorded metric. Empty until the recorder
+/// is installed. The read is lock-free but walks every series and allocates, so
+/// it runs with the GIL released.
+#[pyfunction]
+pub fn snapshot_lance_metrics(py: Python<'_>) -> Vec<PyMetricPoint> {
+    let Some(registry) = REGISTRY.get() else {
+        return Vec::new();
+    };
+    let points = py.detach(|| collect_points(registry));
+    points.into_iter().map(PyMetricPoint::from).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use metrics::HistogramFn;
+
+    fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 {
+        buckets
+            .iter()
+            .find(|(b, _)| b == le)
+            .map(|(_, c)| *c)
+            .unwrap_or_else(|| panic!("no bucket with le={le}"))
+    }
+
+    #[test]
+    fn bucketed_histogram_records_cumulative_buckets() {
+        let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
+        hist.record(0.05); // le=0.1
+        hist.record(0.5); // le=1
+        hist.record(0.5); // le=1
+        hist.record(50.0); // +Inf
+
+        let MetricValue::Histogram {
+            buckets,
+            count,
+            sum,
+        } = hist.snapshot()
+        else {
+            panic!("expected histogram");
+        };
+
+        // Buckets are cumulative (Prometheus `le` semantics).
+        assert_eq!(bucket_count(&buckets, "0.1"), 1);
+        assert_eq!(bucket_count(&buckets, "1"), 3);
+        assert_eq!(bucket_count(&buckets, "10"), 3);
+        assert_eq!(bucket_count(&buckets, "+Inf"), 4);
+        assert_eq!(count, 4);
+        assert!((sum - 51.05).abs() < 1e-9);
+    }
+
+    #[test]
+    fn bucketed_histogram_boundary_is_inclusive() {
+        let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice()));
+        hist.record(1.0); // exactly the bound -> le=1, not +Inf
+        let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
+            panic!("expected histogram");
+        };
+        assert_eq!(bucket_count(&buckets, "1"), 1);
+        assert_eq!(bucket_count(&buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn bucketed_histogram_boundary_is_inclusive_mid_range() {
+        // A value equal to a middle bound lands in that bucket, not the next.
+        let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
+        hist.record(1.0);
+        let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
+            panic!("expected histogram");
+        };
+        assert_eq!(bucket_count(&buckets, "0.1"), 0);
+        assert_eq!(bucket_count(&buckets, "1"), 1);
+        assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1
+        assert_eq!(bucket_count(&buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn recorder_aggregates_counters_with_labels() {
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
+                .increment(2);
+            metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
+                .increment(3);
+            // A distinct label set must produce a separate point, not merge.
+            metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs")
+                .increment(7);
+        });
+
+        let scalar = |attrs: &[(&str, &str)]| {
+            let points = collect_points(&registry);
+            let point = points
+                .into_iter()
+                .find(|p| {
+                    p.name == "test_requests_total"
+                        && attrs
+                            .iter()
+                            .all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v))
+                })
+                .expect("counter recorded for label set");
+            assert_eq!(point.kind, "counter");
+            match point.value {
+                MetricValue::Scalar(v) => v,
+                _ => panic!("expected scalar"),
+            }
+        };
+
+        // Same labels aggregate; distinct labels stay separate.
+        assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9);
+        assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9);
+    }
+
+    #[test]
+    fn recorder_records_gauges() {
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        // Gauges store the f64 bit pattern in a u64; the snapshot must decode it.
+        metrics::with_local_recorder(&recorder, || {
+            metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5);
+        });
+
+        let points = collect_points(&registry);
+        let point = points
+            .iter()
+            .find(|p| p.name == "test_gauge")
+            .expect("gauge recorded");
+        assert_eq!(point.kind, "gauge");
+        assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9));
+    }
+
+    #[test]
+    fn recorder_falls_back_to_default_bounds() {
+        // A histogram with no registered bounds uses DEFAULT_BOUNDS.
+        let name = "test_unregistered_histogram";
+        assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name));
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::histogram!(name).record(0.02);
+        });
+
+        let points = collect_points(&registry);
+        let point = points.iter().find(|p| p.name == name).expect("recorded");
+        let MetricValue::Histogram { buckets, count, .. } = &point.value else {
+            panic!("expected histogram");
+        };
+        assert_eq!(*count, 1);
+        // DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`.
+        assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1);
+        // 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry).
+        assert_eq!(bucket_count(buckets, "0.025"), 1);
+        assert_eq!(bucket_count(buckets, "0.01"), 0);
+        assert_eq!(bucket_count(buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn recorder_uses_registered_histogram_bounds() {
+        let name = "test_recorder_bounds_seconds";
+        HISTOGRAM_BOUNDS
+            .write()
+            .unwrap()
+            .insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice()));
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::histogram!(name).record(0.05);
+            metrics::histogram!(name).record(5.0);
+        });
+
+        let points = collect_points(&registry);
+        let point = points.iter().find(|p| p.name == name).expect("recorded");
+        let MetricValue::Histogram { buckets, count, .. } = &point.value else {
+            panic!("expected histogram");
+        };
+        assert_eq!(*count, 2);
+        assert_eq!(bucket_count(buckets, "0.1"), 1);
+        assert_eq!(bucket_count(buckets, "+Inf"), 2);
+    }
+
+    #[test]
+    fn describe_populates_catalog() {
+        let name = "test_describe_catalog_total";
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder { registry };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::describe_counter!(name, Unit::Count, "a test counter");
+        });
+
+        let catalog = CATALOG.lock().unwrap();
+        let desc = catalog.get(name).expect("described");
+        assert!(matches!(desc.kind, MetricKind::Counter));
+        assert_eq!(desc.description, "a test counter");
+    }
+
+    #[test]
+    fn describe_all_covers_object_store_metrics() {
+        use lance_io::object_store::metrics as os;
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder { registry };
+        metrics::with_local_recorder(&recorder, describe_all);
+
+        let catalog = CATALOG.lock().unwrap();
+        let kind = |name: &str| catalog.get(name).expect("described").kind;
+        // Every emitted object store metric must be described so the OTel
+        // bridge can create an instrument for it, including the gauge and the
+        // retryable counter that a plain request path might never emit.
+        assert!(matches!(kind(os::METRIC_REQUESTS), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_BYTES), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_ERRORS), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_THROTTLE), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_RETRYABLE), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_IN_FLIGHT), MetricKind::Gauge));
+        assert!(matches!(kind(os::METRIC_DURATION), MetricKind::Histogram));
+    }
+}
diff --git a/python/src/schema.rs b/python/src/schema.rs
index db4f7369710..8cdc2115cd1 100644
--- a/python/src/schema.rs
+++ b/python/src/schema.rs
@@ -179,7 +179,8 @@ impl LanceSchema {
             fields: Fields(fields),
             metadata,
         };
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)
+            .map_err(|err| PyValueError::new_err(format!("Failed to reconstruct schema: {err}")))?;
         Ok(Self(schema))
     }
 
diff --git a/python/src/session.rs b/python/src/session.rs
index c91329ec1ee..da3174a9bf9 100644
--- a/python/src/session.rs
+++ b/python/src/session.rs
@@ -3,7 +3,7 @@
 
 use std::sync::Arc;
 
-use pyo3::{pyclass, pymethods};
+use pyo3::{PyResult, pyclass, pymethods};
 
 use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE};
 use lance::session::Session as LanceSession;
@@ -63,6 +63,13 @@ impl Session {
         self.inner.size_bytes()
     }
 
+    /// Return the current size of the index cache in bytes.
+    pub fn index_cache_size_bytes(&self) -> PyResult<u64> {
+        rt().block_on(None, async move {
+            self.inner.index_cache_stats().await.size_bytes as u64
+        })
+    }
+
     /// Return whether the other session is the same as this one.
     pub fn is_same_as(&self, other: &Self) -> bool {
         Arc::ptr_eq(&self.inner, &other.inner)
diff --git a/python/src/transaction.rs b/python/src/transaction.rs
index 1b659395099..e710a36df02 100644
--- a/python/src/transaction.rs
+++ b/python/src/transaction.rs
@@ -7,10 +7,11 @@ use crate::utils::{PyLance, class_name, export_vec, extract_vec};
 use arrow::pyarrow::PyArrowType;
 use arrow_schema::Schema as ArrowSchema;
 use lance::dataset::transaction::{
-    DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap,
-    UpdateMapEntry, UpdateMode,
+    DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction,
+    UpdateMap, UpdateMapEntry, UpdateMode,
 };
 use lance::datatypes::Schema;
+use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
 use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata};
 use pyo3::exceptions::PyValueError;
 use pyo3::types::PySet;
@@ -206,6 +207,128 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataReplacementGroup> {
     }
 }
 
+// The Nth offset in an overlay list positionally maps to the Nth value row in
+// `data_file`, but `RoaringBitmap` stores offsets in ascending order and drops
+// duplicates. A caller-supplied list that isn't strictly ascending would be
+// silently reordered, breaking that mapping, so reject it here instead. This can
+// go away once we expose RoaringBitmap directly to Python (issue #7695).
+fn bitmap_from_sorted_offsets(offsets: Vec<u32>) -> PyResult<RoaringBitmap> {
+    if offsets.windows(2).any(|w| w[0] >= w[1]) {
+        return Err(PyValueError::new_err(
+            "DataOverlayFile.offsets must be strictly ascending with no duplicates; \
+             each offset positionally maps to a value row in data_file",
+        ));
+    }
+    Ok(RoaringBitmap::from_sorted_iter(offsets).expect("offsets verified strictly ascending"))
+}
+
+impl FromPyObject<'_, '_> for PyLance<DataOverlayFile> {
+    type Error = PyErr;
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
+        let data_file = ob.getattr("data_file")?.extract::<PyLance<DataFile>>()?.0;
+        let offsets = ob.getattr("offsets")?;
+
+        // A flat list of offsets is a dense overlay (one coverage shared by every
+        // field); a list of per-field lists is a sparse overlay. Differentiate by
+        // shape, trying the dense form first.
+        let coverage = if let Ok(shared) = offsets.extract::<Vec<u32>>() {
+            OverlayCoverage::dense(bitmap_from_sorted_offsets(shared)?)
+        } else if let Ok(per_field) = offsets.extract::<Vec<Vec<u32>>>() {
+            OverlayCoverage::sparse(
+                per_field
+                    .into_iter()
+                    .map(bitmap_from_sorted_offsets)
+                    .collect::<PyResult<Vec<_>>>()?,
+            )
+        } else {
+            return Err(PyValueError::new_err(
+                "DataOverlayFile.offsets must be a list of ints (dense coverage shared by \
+                 every field) or a list of per-field int lists (sparse coverage)",
+            ));
+        };
+
+        // Present (and preserved) when round-tripping an existing fragment's
+        // overlays; None/0 when creating an overlay to commit, since the
+        // DataOverlay commit stamps the effective version.
+        let committed_version = ob
+            .getattr("committed_version")?
+            .extract::<Option<u64>>()?
+            .unwrap_or(0);
+
+        Ok(Self(DataOverlayFile {
+            data_file,
+            coverage,
+            committed_version,
+        }))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayFile> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let namespace = py
+            .import(intern!(py, "lance"))
+            .and_then(|module| module.getattr(intern!(py, "LanceOperation")))
+            .expect("Failed to import LanceOperation namespace");
+
+        let data_file = PyLance(&self.0.data_file).into_pyobject(py)?;
+        let cls = namespace
+            .getattr("DataOverlayFile")
+            .expect("Failed to get DataOverlayFile class");
+
+        let committed_version = self.0.committed_version;
+
+        // Mirror the read side: a dense overlay becomes a flat list of offsets, a
+        // sparse overlay a list of per-field lists.
+        match &self.0.coverage {
+            OverlayCoverage::Shared(bitmap) => {
+                let offsets: Vec<u32> = bitmap.iter().collect();
+                cls.call1((data_file, offsets, committed_version))
+            }
+            OverlayCoverage::PerField(bitmaps) => {
+                let offsets: Vec<Vec<u32>> = bitmaps.iter().map(|b| b.iter().collect()).collect();
+                cls.call1((data_file, offsets, committed_version))
+            }
+        }
+    }
+}
+
+impl FromPyObject<'_, '_> for PyLance<DataOverlayGroup> {
+    type Error = PyErr;
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
+        let fragment_id = ob.getattr("fragment_id")?.extract::<u64>()?;
+        let overlays = extract_vec(&ob.getattr("overlays")?)?;
+        Ok(Self(DataOverlayGroup {
+            fragment_id,
+            overlays,
+        }))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayGroup> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let namespace = py
+            .import(intern!(py, "lance"))
+            .and_then(|module| module.getattr(intern!(py, "LanceOperation")))
+            .expect("Failed to import LanceOperation namespace");
+
+        let fragment_id = self.0.fragment_id;
+        let overlays = export_vec(py, self.0.overlays.as_slice())?;
+
+        let cls = namespace
+            .getattr("DataOverlayGroup")
+            .expect("Failed to get DataOverlayGroup class");
+        cls.call1((fragment_id, overlays))
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct PyUpdateMode(pub UpdateMode);
 
@@ -295,7 +418,7 @@ impl FromPyObject<'_, '_> for PyLance<Operation> {
                     updated_fragments,
                     new_fragments,
                     fields_modified,
-                    merged_generations: vec![],
+                    compacted_sstables: vec![],
                     fields_for_preserving_frag_bitmap,
                     update_mode,
                     inserted_rows_filter: None,
@@ -350,6 +473,13 @@ impl FromPyObject<'_, '_> for PyLance<Operation> {
 
                 Ok(Self(op))
             }
+            "DataOverlay" => {
+                let groups = extract_vec(&ob.getattr("groups")?)?;
+
+                let op = Operation::DataOverlay { groups };
+
+                Ok(Self(op))
+            }
             "Project" => {
                 let schema = extract_schema(&ob.getattr("schema")?)?;
 
@@ -487,6 +617,13 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> {
                     .expect("Failed to get DataReplacement class");
                 cls.call1((replacements,))
             }
+            Operation::DataOverlay { groups } => {
+                let groups = export_vec(py, groups.as_slice())?;
+                let cls = namespace
+                    .getattr("DataOverlay")
+                    .expect("Failed to get DataOverlay class");
+                cls.call1((groups,))
+            }
             Operation::Delete {
                 updated_fragments,
                 deleted_fragment_ids,
diff --git a/python/src/utils.rs b/python/src/utils.rs
index 4f7d6d7dde2..edc376d35c8 100644
--- a/python/src/utils.rs
+++ b/python/src/utils.rs
@@ -26,7 +26,6 @@ use lance::Result;
 use lance::datatypes::Schema;
 use lance_arrow::FixedSizeListArrayExt;
 use lance_file::previous::writer::FileWriter as PreviousFileWriter;
-use lance_index::scalar::IndexWriter;
 use lance_index::vector::hnsw::{HNSW, builder::HnswBuildParams};
 use lance_index::vector::kmeans::{
     KMeans as LanceKMeans, KMeansAlgoFloat, KMeansParams, compute_partitions,
@@ -255,7 +254,7 @@ impl Hnsw {
         rt().block_on(Some(py), async {
             let batch = self.hnsw.to_batch()?;
             let metadata = batch.schema_ref().metadata().clone();
-            writer.write_record_batch(batch).await?;
+            writer.write(&[batch]).await?;
             writer.finish_with_metadata(&metadata).await?;
             Result::Ok(())
         })?
diff --git a/python/uv.lock b/python/uv.lock
index a8a7febeea4..fe581b05222 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -9,27 +9,18 @@ resolution-markers = [
     "python_full_version < '3.11'",
 ]
 
-[[package]]
-name = "absl-py"
-version = "2.3.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" },
-]
-
 [[package]]
 name = "aiohappyeyeballs"
-version = "2.6.1"
+version = "2.7.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" },
+    { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" },
 ]
 
 [[package]]
 name = "aiohttp"
-version = "3.14.0"
+version = "3.14.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiohappyeyeballs" },
@@ -42,126 +33,126 @@ dependencies = [
     { name = "typing-extensions", marker = "python_full_version < '3.13'" },
     { name = "yarl" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/f0/f81190ba488cd106c2fc6d92680e56bb223bbbbf1e6908c2617011290112/aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c", size = 760606, upload-time = "2026-06-01T19:36:39.054Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/54/444d37eebf0f15db661ca44ec7caf93962f3c5ca92eb4c9a5d888b70aaa2/aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2", size = 514677, upload-time = "2026-06-01T19:36:42.408Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/d1/da280e23321c132c0a3fa7c8cc2830621d79174edc64c829443346489a36/aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd", size = 510155, upload-time = "2026-06-01T19:36:44.072Z" },
-    { url = "https://files.pythonhosted.org/packages/09/b8/2e36d54d0991ec5bba451444004591ee0af58cb1662a3a81c562878b9c1f/aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869", size = 1699947, upload-time = "2026-06-01T19:36:45.762Z" },
-    { url = "https://files.pythonhosted.org/packages/57/95/a31d8ea1a0b9ecc084f5a7dd0b431ce64ef585918bb7bdc82afe11843877/aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4", size = 1664364, upload-time = "2026-06-01T19:36:47.653Z" },
-    { url = "https://files.pythonhosted.org/packages/01/f6/5de3ddffc87a9e8d09b3be38fbd6dd1a736b2ad477a7e787dcb85f57f338/aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb", size = 1761186, upload-time = "2026-06-01T19:36:49.355Z" },
-    { url = "https://files.pythonhosted.org/packages/33/8c/03c5438ec35d7e3a4f33fe895d6c3ec7540a7cec46065f21851211e1ee4d/aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca", size = 1849727, upload-time = "2026-06-01T19:36:51.478Z" },
-    { url = "https://files.pythonhosted.org/packages/22/32/5a05303b0874458920b73f48b8779cc3a93d503f121b38dcc0456dbd698c/aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f", size = 1708197, upload-time = "2026-06-01T19:36:53.241Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/62/478f169488d61414c0a05e7fe423b59ae3d9dcc933d1f0e4acc2c5d5bc3e/aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6", size = 1578147, upload-time = "2026-06-01T19:36:55.154Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/af/b20af85765658972d3337834bd5eebba91b962794f2b4fc3e0ee8c85c0e1/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b", size = 1665836, upload-time = "2026-06-01T19:36:56.94Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/a3/771879cfd59948f4544b172189048905feff802f20f1c6c5411e998a3e06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15", size = 1680335, upload-time = "2026-06-01T19:36:58.642Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/16/582e36ad1d32133cd40659f3bc98e71c22179665a1cfbbb4713bce339c06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce", size = 1731180, upload-time = "2026-06-01T19:37:00.583Z" },
-    { url = "https://files.pythonhosted.org/packages/11/bc/80708fe3f64a07a2c306a42fc7b009118a952709761d215f6d1b4c57195b/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7", size = 1565805, upload-time = "2026-06-01T19:37:02.446Z" },
-    { url = "https://files.pythonhosted.org/packages/57/8f/8d25897f8273a32fe4ad40a8885eec4f397377ed46e8e383078169f60316/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b", size = 1742496, upload-time = "2026-06-01T19:37:04.222Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/7d/c341d32ab2dec56c8478740695743dc6c21b383cace9376a3eab16311a07/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25", size = 1691240, upload-time = "2026-06-01T19:37:06.277Z" },
-    { url = "https://files.pythonhosted.org/packages/37/0f/a81207dd7a2d4a4f645b3a3f8b5a1da1159dc63117ffb137b698fd6df50f/aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f", size = 454686, upload-time = "2026-06-01T19:37:07.96Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/ae/842357f2afb9c915715c6f5775239d987f5d0f845abf7675fa794e0a9d40/aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594", size = 478677, upload-time = "2026-06-01T19:37:09.652Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/d1/330fb22c9535ec177b52396905131c6e39447244b6ca876262939af668ef/aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a", size = 450364, upload-time = "2026-06-01T19:37:11.279Z" },
-    { url = "https://files.pythonhosted.org/packages/67/47/7727bfe8db93f8835a001bd4359d8480cc68d1259b8bce334668f8be97bd/aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a", size = 759147, upload-time = "2026-06-01T19:37:12.918Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/f2/cd3fedff6fade73d71df9ec908c210cec518ef90fd00289250684b90aecf/aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803", size = 513705, upload-time = "2026-06-01T19:37:14.633Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/fe/49746b6b610144a06323bebd8e1211a390310d8c69b98dd6d52df341bc3e/aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e", size = 509627, upload-time = "2026-06-01T19:37:16.385Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/3f/28f2f6cf3d5c0e7b01b27140d0e7873fd11fb341169ad3ce78ad04aba628/aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903", size = 1769293, upload-time = "2026-06-01T19:37:18.067Z" },
-    { url = "https://files.pythonhosted.org/packages/97/6f/2e5f1b525d5474b12b3c60abf733a755845f3bceff21542081ada515f837/aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb", size = 1732363, upload-time = "2026-06-01T19:37:20.138Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/ce/596120faa85ca7b19cd061e3f2f3be23aa8f11a0aedf9191db9e0da1bd76/aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2", size = 1840375, upload-time = "2026-06-01T19:37:22.104Z" },
-    { url = "https://files.pythonhosted.org/packages/72/3c/a7ffe05a757a4a7867643da69357ec41f506879fbd1b231d2ed90af246b2/aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81", size = 1921484, upload-time = "2026-06-01T19:37:24.068Z" },
-    { url = "https://files.pythonhosted.org/packages/93/fa/2c861170bbd4a491de93a69e081db1d971092569e0d593a98ef62c384dc1/aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee", size = 1774153, upload-time = "2026-06-01T19:37:26.256Z" },
-    { url = "https://files.pythonhosted.org/packages/9d/da/1d2f5a165f47ec9b1f69d37b8b977fdc4d501aa72ffb7930db27bb9e49ea/aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d", size = 1632569, upload-time = "2026-06-01T19:37:28.192Z" },
-    { url = "https://files.pythonhosted.org/packages/46/1d/7a6e295c4257252f70f69e90864fdad74b6a1293054fb3f9e65a15de6d63/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00", size = 1740325, upload-time = "2026-06-01T19:37:30.08Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/7e/e1899b1ca3ec62f1eab2a5cbde14039b97493f7f53eb88d9b668562ffa8d/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026", size = 1748691, upload-time = "2026-06-01T19:37:32.211Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/54/4e6b61c1fe7d3433f82bcc6bd7e4d7c683a742a10c9b12a025fd3695c047/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86", size = 1814477, upload-time = "2026-06-01T19:37:34.173Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/38/86fd51be2e08d8e45c83d879d255f10391903cd9fe2a16512f7591a15873/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f", size = 1623393, upload-time = "2026-06-01T19:37:36.281Z" },
-    { url = "https://files.pythonhosted.org/packages/78/49/466e947a42a88ee23c486d036e7e5d1b097f1bafd8084ad9c9a0a92f0f43/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93", size = 1824097, upload-time = "2026-06-01T19:37:38.421Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/89/35f3410bc284682338a1be6b6ea0c5abfa05f063942cfaa9256608440434/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996", size = 1764790, upload-time = "2026-06-01T19:37:40.755Z" },
-    { url = "https://files.pythonhosted.org/packages/42/80/2d4291bd5724d3d17e5951aff5a3e02281483fb47295f0788276ee66cd73/aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae", size = 454176, upload-time = "2026-06-01T19:37:42.837Z" },
-    { url = "https://files.pythonhosted.org/packages/59/ed/41d0ad4f6ececffc32bdf1f7b494e5498f7ca5c849ea2e3cc9bbd1668251/aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5", size = 479334, upload-time = "2026-06-01T19:37:44.776Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/86/c0b5e305c770053f8c3d069bb52b8196917ba91949d1962d52eb307fb0d2/aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43", size = 450262, upload-time = "2026-06-01T19:37:46.461Z" },
-    { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" },
-    { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" },
-    { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" },
-    { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" },
-    { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" },
-    { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" },
-    { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" },
-    { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" },
-    { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" },
-    { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" },
-    { url = "https://files.pythonhosted.org/packages/21/61/d11f7d9a3144bffe825247d6367cd93053666da50b94707c9129c78868d5/aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127", size = 502399, upload-time = "2026-06-01T19:38:25.955Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/9b/a7e317625d36356844f8bb022cabd305b541f968856cc3c2e0b58e53ee6e/aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981", size = 510068, upload-time = "2026-06-01T19:38:27.828Z" },
-    { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/6a/42e030a46743841414402a3b00cd3d78419055e86c66fb5822c14b5abfc6/aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61", size = 1729674, upload-time = "2026-06-01T19:38:44.79Z" },
-    { url = "https://files.pythonhosted.org/packages/34/26/3199beb415202e3108e7b83ecebe10914d806d33fb9860c3e4aa60a19be3/aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277", size = 1798808, upload-time = "2026-06-01T19:38:47.01Z" },
-    { url = "https://files.pythonhosted.org/packages/bd/94/b9b6fcf0ee17c21d0d19fb8c22bf83ad18f82e702a9c3bd901a868f5e446/aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882", size = 1891921, upload-time = "2026-06-01T19:38:49.233Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" },
-    { url = "https://files.pythonhosted.org/packages/21/2a/45be91ad1b860508557448d4cc2e165a2ee68dd865657b73bf66cc5a00fb/aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988", size = 1579554, upload-time = "2026-06-01T19:38:54.508Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/e4/1f1c8acbb3acd5c8f795473b92c9c3d44eb60a5692c6104256c8a1c83a0c/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3", size = 1749803, upload-time = "2026-06-01T19:38:59.367Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/c8/c45ea6e7ed84cebba939b9c334498a045ba19d79c61b0110df5f21580de3/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac", size = 1765023, upload-time = "2026-06-01T19:39:01.651Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/a1/a932941784432962fe390e1066823aaef64b4e5ac9fa595df57b5fe472a9/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e", size = 1571671, upload-time = "2026-06-01T19:39:04.044Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/01/e1280feac522597a4d46eb67a0cdfa053cfae263033030b761ab146f29fb/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec", size = 1789904, upload-time = "2026-06-01T19:39:06.294Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" },
-    { url = "https://files.pythonhosted.org/packages/af/cc/c122eabd7a1b7e0c9bbdd6be60e4715905b858399145d9df872bb94f1427/aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456", size = 448656, upload-time = "2026-06-01T19:39:11.171Z" },
-    { url = "https://files.pythonhosted.org/packages/41/a5/bab07d79848a00eedd8ed979ccb302aaea3ac6eb9fa16bd0ed87135869b4/aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11", size = 475803, upload-time = "2026-06-01T19:39:13.439Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/a0/f03ade8566c153666a3871afccbedf6d99911da006325e1fc6cf72a2de99/aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b", size = 443889, upload-time = "2026-06-01T19:39:15.945Z" },
-    { url = "https://files.pythonhosted.org/packages/28/03/5f36ab196a88ba5e9648ae5643e6531e67a3a8c0e96f9c6510ff41540fec/aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f", size = 503330, upload-time = "2026-06-01T19:39:18.195Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/ce/8b49ec2f30f68e02f314f4832186cd45e583360a5a386058be36855d23b6/aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42", size = 509822, upload-time = "2026-06-01T19:39:20.396Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" },
-    { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" },
-    { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/28/24e1409e605a9aa5d84abe0e2acb365354b70ae56d40948101cabe3341ab/aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f", size = 1705862, upload-time = "2026-06-01T19:39:38.968Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/d0/e5eb3ff1daeaf644c7e36a957517672494122628e067c38b263fa04eda77/aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3", size = 1798909, upload-time = "2026-06-01T19:39:41.334Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/ba/8943f906f0570342886ababb9a722a44e360f786a028c5e0b0e29e3f735b/aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6", size = 1868892, upload-time = "2026-06-01T19:39:43.807Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/62/da182e5910ab912b2e88aa919b61a16046a37a95714a5795b02eb57b2d18/aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c", size = 1578775, upload-time = "2026-06-01T19:39:48.902Z" },
-    { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/55/0e2732ca598c7a4dfe8a775662376d0ca2977cb1030e48386d4da5d9a456/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4", size = 1715016, upload-time = "2026-06-01T19:39:53.807Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/96/f0b73730798c9ca525afc30b39f1f81bbe24e245d9654c54d3b39d63212d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c", size = 1763810, upload-time = "2026-06-01T19:39:56.31Z" },
-    { url = "https://files.pythonhosted.org/packages/71/cc/11acb6c4518f448323405a7312b6f255d0f974a34373ad1db7633c4aadc8/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3", size = 1573064, upload-time = "2026-06-01T19:39:58.718Z" },
-    { url = "https://files.pythonhosted.org/packages/de/2d/28c31dde0a7dc98c0ee7d0da2ddcec3f7688c4fc131e5989e278d0c03c0a/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb", size = 1775765, upload-time = "2026-06-01T19:40:01.195Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/44/6126116fd8a316b712bb615660b855c78466bb67ba1bb1742427eafcf7ac/aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d", size = 453684, upload-time = "2026-06-01T19:40:06.277Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/d7/eff4c58a88c5cac5e38b55f44fb8a6d3929c3cbd77356e383e094d3220bd/aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7", size = 481758, upload-time = "2026-06-01T19:40:08.653Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/ed/17b5bd9fbcb46e688f02e572f517754a9a75831e7b54702f027761dc4fa5/aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6", size = 450557, upload-time = "2026-06-01T19:40:11.03Z" },
-    { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" },
-    { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" },
-    { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" },
-    { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/85/ce79bab0310d2e3fd2d7bc7e44412abeff7c8338f8a21dd0f2f1714989e5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a", size = 1778813, upload-time = "2026-06-01T19:40:23.726Z" },
-    { url = "https://files.pythonhosted.org/packages/05/54/ba62ac2d1bc87e010aad23751e383b8794e45d931df67677313a2da78823/aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936", size = 1899969, upload-time = "2026-06-01T19:40:26.406Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/82/7cc7907725d83a19f31551334061e1ab8e108b1d7ac52632a2a844a4acb5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e", size = 1991771, upload-time = "2026-06-01T19:40:29.061Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/ae/3839726cd49150a53ed340cc24ce5ba09d4c2117020ef9d45542bec5eb2f/aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a", size = 1665437, upload-time = "2026-06-01T19:40:35.01Z" },
-    { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" },
-    { url = "https://files.pythonhosted.org/packages/98/02/a5a7a2524f92d3911761b405a7c067c751891942144adc13e2ad79611e39/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce", size = 1816907, upload-time = "2026-06-01T19:40:40.46Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/76/a8b9f0d09234d516af9f2d7dd715557f33b5da3b0b56ead41d1170e86e3c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c", size = 1840382, upload-time = "2026-06-01T19:40:43.48Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/8e/140e715a0a4bbc211979ea30ec8396ad2ed5bf90ab87d8058fc4668b1923/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7", size = 1659497, upload-time = "2026-06-01T19:40:46.265Z" },
-    { url = "https://files.pythonhosted.org/packages/10/c7/7ba5de8af9650b9767b063c675427b8685f43fa7ce563673a7bc3af60f08/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928", size = 1870829, upload-time = "2026-06-01T19:40:49.583Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" },
-    { url = "https://files.pythonhosted.org/packages/39/98/31b9ad9fbc01f0075ee7221002df5fd2d10b647f451ca5f30edc802d9dd6/aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6", size = 490597, upload-time = "2026-06-01T19:40:54.937Z" },
-    { url = "https://files.pythonhosted.org/packages/59/1f/299b21441c8de42ff70fddc7cfe65e92f810abcf740739a09b56f7835364/aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2", size = 525789, upload-time = "2026-06-01T19:40:57.306Z" },
-    { url = "https://files.pythonhosted.org/packages/70/11/7f83fcba9ee05d4c54d61b3f8104da0d43a59adac44dd28effc0c9a10422/aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24", size = 467399, upload-time = "2026-06-01T19:40:59.993Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" },
+    { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" },
+    { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" },
+    { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" },
+    { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" },
+    { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" },
+    { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" },
+    { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" },
+    { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" },
+    { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" },
+    { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" },
+    { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" },
+    { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" },
+    { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" },
+    { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" },
+    { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" },
+    { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" },
+    { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" },
+    { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" },
+    { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" },
+    { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" },
+    { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" },
+    { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" },
+    { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" },
+    { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" },
+    { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" },
+    { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" },
+    { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" },
+    { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" },
+    { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" },
+    { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" },
+    { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" },
+    { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" },
+    { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" },
+    { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" },
+    { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" },
+    { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" },
+    { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" },
 ]
 
 [[package]]
@@ -187,78 +178,94 @@ wheels = [
 ]
 
 [[package]]
-name = "arro3-core"
-version = "0.6.5"
+name = "anyio"
+version = "4.14.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version < '3.12'" },
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "idna" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/8a/24b35cf01a68621f5f07e3191ca96f70a145022ca367347266901eb504a7/arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae", size = 2678116, upload-time = "2025-10-13T23:09:04.198Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/7a/4398bb0582fb22d575f256f2b9ac7be735c765222cc61fb214d606bdb77c/arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a", size = 2383214, upload-time = "2025-10-13T23:09:06.841Z" },
-    { url = "https://files.pythonhosted.org/packages/82/3f/a321501c5da4bf3ff7438c3e5eb6e63bcecb5630c0f4a89a017cbfa8e4a0/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6584a3d28007740afcef1e301332876e2b785bd8edd59a458a6bc9b051bce052", size = 2883536, upload-time = "2025-10-13T23:09:08.877Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/50/1d1e55b9a8c4cf2fdeb954947aa135010554a3333b709e8cad3d5d084be2/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e0af4789618f02bead4a0cd4d0a54abd9c8aa4fcedf9872b4891d2e3e984161", size = 2908828, upload-time = "2025-10-13T23:09:10.958Z" },
-    { url = "https://files.pythonhosted.org/packages/12/75/b4b1de1ccb17890bada9a3f4131cf3137f145d5d10490db51de6b8799926/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c73f212e549e9b6d11cfe3f14bbf3fba9d0891426afb5916688d16d0df724085", size = 3145458, upload-time = "2025-10-13T23:09:13.275Z" },
-    { url = "https://files.pythonhosted.org/packages/08/4f/f42ce1840490fd0863bfbc56f28eaaec3bcb4eb322079af9c070111657e5/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f88f62e4e276a9e84f250722d2e5ffc078af9a3f67ac691f572a0e05dd6095", size = 2775793, upload-time = "2025-10-13T23:09:15.342Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/aa/9637efc8d8733c34bedef44e5b2c170dea14d15ab56b3566d8d7963c2616/arro3_core-0.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:b2635e4c227f25ff8784dc8efb38cb7c1674646cfdc68ded53f2426289885f0e", size = 2516697, upload-time = "2025-10-13T23:09:17.584Z" },
-    { url = "https://files.pythonhosted.org/packages/60/84/1fcfadf956bc25eb5251b1ea7a7099f05198a55764635d2fc9ceafdbdbd1/arro3_core-0.6.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a5f3e936686bcd8542fafc94c68fdb23ec42d1d51a4777967ae815c90aff7296", size = 3023625, upload-time = "2025-10-13T23:09:21.556Z" },
-    { url = "https://files.pythonhosted.org/packages/58/d0/52d0cb3c0dfa8e94ba2118b7e91a70da76d6ede9de4e70374f831f38cfdf/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:705c32fec03dadc08f807d69ce557882005d43eb20ec62699f7036340f0d580f", size = 2701346, upload-time = "2025-10-13T23:09:25.031Z" },
-    { url = "https://files.pythonhosted.org/packages/69/bf/42a6f6501805c31cb65d8a6e3379eeec4fa6c26dc07c9ce894f363ccad1c/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:56d8166235a4c54e4f7ba082ec76890c820fa8c1b6c995ec59cead62a9698e59", size = 3153207, upload-time = "2025-10-13T23:09:28.254Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/e5/41fdee468b33759b42958347c2d70b0461bf8f70ba1762a94cdf2e9b0142/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1ba43ba9081c00767083195222b6be74913de668296f55599658c4b0bb7cd327", size = 3105033, upload-time = "2025-10-13T23:09:31.545Z" },
-    { url = "https://files.pythonhosted.org/packages/03/e0/b6d733b4540c05bac546162e045b547031f4d88c67b7c864929d9bce29ad/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f5df13c6742e3f0b494cfe9025dccdc8426a74cc9e3e5a1239311e07a4b24e0", size = 2954793, upload-time = "2025-10-13T23:09:34.988Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/34/8353ba79c8d0498eaacc077d58b384ef785e0b69c9cbff7c2580136b8fe3/arro3_core-0.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:34676b728178236df63c9ea10b21432392d4b5bb51e2030e77c68eed4dede2ad", size = 2837495, upload-time = "2025-10-13T23:09:38.539Z" },
-    { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944, upload-time = "2025-10-13T23:09:45.266Z" },
-    { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376, upload-time = "2025-10-13T23:09:48.737Z" },
-    { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568, upload-time = "2025-10-13T23:09:51.95Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223, upload-time = "2025-10-13T23:09:55.387Z" },
-    { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862, upload-time = "2025-10-13T23:09:59.035Z" },
-    { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442, upload-time = "2025-10-13T23:10:02.134Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454, upload-time = "2025-10-13T23:10:05.192Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984, upload-time = "2025-10-13T23:10:08.518Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472, upload-time = "2025-10-13T23:10:11.976Z" },
-    { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099, upload-time = "2025-10-13T23:10:15.374Z" },
-    { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677, upload-time = "2025-10-13T23:10:18.828Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445, upload-time = "2025-10-13T23:10:22.345Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985, upload-time = "2025-10-13T23:10:25.515Z" },
-    { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931, upload-time = "2025-10-13T23:10:29.893Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900, upload-time = "2025-10-13T23:10:33.238Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149, upload-time = "2025-10-13T23:10:36.547Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477, upload-time = "2025-10-13T23:10:40.232Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522, upload-time = "2025-10-13T23:10:43.413Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426, upload-time = "2025-10-13T23:10:46.926Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759, upload-time = "2025-10-13T23:10:50.281Z" },
-    { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191, upload-time = "2025-10-13T23:10:53.776Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963, upload-time = "2025-10-13T23:10:57.163Z" },
-    { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" },
-    { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" },
-    { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788, upload-time = "2025-10-13T23:11:56.965Z" },
-    { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809, upload-time = "2025-10-13T23:12:00.175Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818, upload-time = "2025-10-13T23:12:03.721Z" },
-    { url = "https://files.pythonhosted.org/packages/58/7f/16f741e1d49ba5c5a893ce6f8eb0283d64bc68d6cc9e07ac62f96eaadfae/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:def7b0065a684d6f903a658d2567da47e2fcecde716e0b34eff4d899c6468c8d", size = 2907503, upload-time = "2025-10-13T23:12:07.066Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/45/2eb7972e0bbec0ee0ab22b0f166ec1ea74b53bd76c93a18ced434713e495/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbfe2f2d4d0d393833cd6a4bd9c15266a02307a3028f159155a1c536469c3ae7", size = 3143706, upload-time = "2025-10-13T23:12:10.492Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/af/b78e28842faa675e4e6c4d82e861accf21ac08bbab80a65fa80c578f80a1/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a191a3e4f72c34f7ace7724a94f2d90b06c804a6cbece4ae0f18d36325479cf3", size = 2775462, upload-time = "2025-10-13T23:12:14.026Z" },
-    { url = "https://files.pythonhosted.org/packages/45/df/950e57e4915e0457acadaaca13c4423d5e2652e403135eb7606d5e6e5443/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:e3f6ab4c6ea96c451eff72aa6c5b9835a0ea8a9847cfe3995c88cce0c7701fb5", size = 2516212, upload-time = "2025-10-13T23:12:17.548Z" },
-    { url = "https://files.pythonhosted.org/packages/07/73/821640d0827a829ed2565c2d4812080ab7fb86f0d271b462f9b37e6d946e/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27df5239835330299636a02977f2cb34d5c460cc03b2ae1d6ab6a03d28051b08", size = 3023342, upload-time = "2025-10-13T23:12:21.308Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/30/51302d2f4d1b627dd11e2be979f2c48550b782d8d58d0378316342e284a8/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:71dce89c0e91be4cfb42591f03809235bbc374c396e08acdf93c4d85b09e40f5", size = 2700740, upload-time = "2025-10-13T23:12:24.968Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/e8/0c8a345a013bb64abea60b4864bacc01e43b8699b8874794baec9c8a7e76/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:d380c28f85568ed99c1686fb9d64b5a811d76d569f367cbec8ef7e58f6e2fdf9", size = 3152749, upload-time = "2025-10-13T23:12:28.393Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/42/003b30c4da394366d5967a5b993f7471a74182c983d8f757891b3dd5d594/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:8e359c0c4fe9992f5a863a4a31502ea58eb2f92988fc2e501850540b3eff0328", size = 3104676, upload-time = "2025-10-13T23:12:31.711Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405, upload-time = "2025-10-13T23:12:35.328Z" },
-]
-
-[[package]]
-name = "astunparse"
-version = "1.6.3"
+sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
+]
+
+[[package]]
+name = "arro3-core"
+version = "0.8.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "six" },
-    { name = "wheel" },
+    { name = "typing-extensions", marker = "python_full_version < '3.12'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/c0/c8/fc5bacb6fc264dc61e46d4832f690015b7f6c693ff5dea8a1e53b63cb772/arro3_core-0.8.1.tar.gz", hash = "sha256:1df54a8e2c14a877f291d90de65f00bafe9cc6d5958417ff749f178f059dcd39", size = 93684, upload-time = "2026-06-11T18:01:37.508Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ea/18/81ff17882bb43bcef22bfc67d5b9522ee52f39b0bc6516eda718f99fb1aa/arro3_core-0.8.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:88b15a937ebe7e64f63cbbd1134d18daa3cb3f769e936a1f0284136375707ebb", size = 3077128, upload-time = "2026-06-11T17:59:25.637Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0501662cb3c7c6a05dcd6b998db55f9a97743aa571da42203733366c42166a7a", size = 2798583, upload-time = "2026-06-11T17:59:27.193Z" },
+    { url = "https://files.pythonhosted.org/packages/39/80/62e19c6935d5633452a76354881be62f7a25f1bc7d8ad2e74faf3c68790a/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5674a052c68721963f462b10ce92596a7e85cfacd3fbdc3839633182844f6f3c", size = 3270157, upload-time = "2026-06-11T17:59:28.785Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/7d/0eadbcc314d61cad40679878bab76ae224a476059097af271051e4f36024/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ebc5e8917ccc638f276292532e7a59dad3d89ba8a26e874e596a6bc4ffb0ca4b", size = 3404256, upload-time = "2026-06-11T17:59:30.276Z" },
+    { url = "https://files.pythonhosted.org/packages/76/6f/6b2ed8c8c0f4bff723e9485ac64daf3d34e424f3d266517e002644b9171d/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f30c53a51934884416b752e95ef50b1958d6b55de6a386e919c202ceec7d4ff", size = 3467310, upload-time = "2026-06-11T17:59:31.889Z" },
+    { url = "https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31b966b0f9b2a6fb3f285339ed5ec4eac69591bbb92a9fdb2cf80ad995b759af", size = 3190458, upload-time = "2026-06-11T17:59:33.385Z" },
+    { url = "https://files.pythonhosted.org/packages/73/63/fd6af88fe75a1316f7742ac11937fb9ab94ffbc582e9e722b807cf2957fa/arro3_core-0.8.1-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:f27615f05e55e7c277b4386fd18356d7b052e0c92be66cc13cf823286d94b141", size = 2950698, upload-time = "2026-06-11T17:59:34.97Z" },
+    { url = "https://files.pythonhosted.org/packages/40/a6/5c3f9c4a8f0f8c510e6b0fea41329f704d6be569cc52f7479c179a84d196/arro3_core-0.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f6c9e50b3bc594260cbcc94e4b9fff590c5fd7f35a6c5729ecc6a6ff0a117981", size = 3398996, upload-time = "2026-06-11T17:59:36.369Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/0c/b52e17b284e9b0b3cb0c402071e1057bf422309407e7f9c3bc80818ee581/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97584eb695f6d9a2cb5e66aa4866fdd17316c313d1724257660bcb0f37a59fb1", size = 3128930, upload-time = "2026-06-11T17:59:37.914Z" },
+    { url = "https://files.pythonhosted.org/packages/58/99/6e6b15a8e9b6ea8995f2b0ccd01a91fd3e72af74ef45b856b7675bb58b04/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:35dfbc1c19afb63371ae23bd653202b57fd99b57f17ed1561849c4b5994b63d9", size = 3547956, upload-time = "2026-06-11T17:59:39.327Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/3e/82d1df9edcd26632c5b6271ebbc6c825307abb76dc9c8de04094510b3dc6/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2dd781426d776fad17dba1c991df422d071d22e59cfd0db80b104648b6c96040", size = 3505514, upload-time = "2026-06-11T17:59:41.219Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/c8/ffc2d069ba82444fbeb0a41b24a4c51ddfe99935b11c8572ce3c1911c19f/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:540f85f0b6a31820347d069c217b002488fe60f8e509d134bd1e0bc2eeffbfff", size = 3405220, upload-time = "2026-06-11T17:59:43.275Z" },
+    { url = "https://files.pythonhosted.org/packages/df/12/d7de53951ed8b89bc3f390b416e25e92dd765bc2faa13eedcd467248bc24/arro3_core-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:507c4a58c993af697eb2e7b324c1f2c4ced22a72c85e9c2adbc23d28c6a6c20b", size = 3357259, upload-time = "2026-06-11T17:59:44.961Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/75/29517738623cccba1d60d0aa65b896bdd046ab4a82c5cd5a9fca115f6d7a/arro3_core-0.8.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:64d3ea60da4279e9c6a9b2e60abf7f4fefb6643544ecb2dfde899f72f1f91bad", size = 3085640, upload-time = "2026-06-11T17:59:46.597Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/b5/bff1842ed8dbb2134b004f78c52d977b52a6df1d9389b1284aad02bf4d93/arro3_core-0.8.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:463db0b7f698abc19f4274d6df697560bc9c19463cfdbd01094bb0fa6ddd1206", size = 2800705, upload-time = "2026-06-11T17:59:48.242Z" },
+    { url = "https://files.pythonhosted.org/packages/32/20/ef60404d5008f84bca50510f65d7acfe26812e61638ee19a416bf3f11530/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:881afcb9c83334ac26498b429265223c676e028cd5d88fb0c909578f8e0330de", size = 3272103, upload-time = "2026-06-11T17:59:49.641Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/8e/6687efc8e0414cfde6281b1f4ea3b5b44aeb318584e39113d645d1f913a2/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1dc03e1e5965528bfc7e26a6ee857e3b6fc5f201ff530a03dc731b9f3fb14fd", size = 3405750, upload-time = "2026-06-11T17:59:51.037Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/38/2567f26cd041387a2c1f381f1757b5246184de22dd8bd9dbbcb91a1b3586/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:410cdca92392be39a5580059ebf9a6f89a0410af6dc10a8baa76e9d511fa6624", size = 3467290, upload-time = "2026-06-11T17:59:52.672Z" },
+    { url = "https://files.pythonhosted.org/packages/85/38/2c5af3a3e8c806188f1b3a651cbaa6c22d1f094c41e82900d40219c2b337/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4082c6bbff7f164619d99dffcbc2313ed69024653a9e58d9f94cc65d9226f6c", size = 3195582, upload-time = "2026-06-11T17:59:54.536Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/8d/d7d8686901a273743c53aae98f898f00caaedea807e28854200f2a7365e4/arro3_core-0.8.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:40da2ee0701f217cd482e61228023ee9d8993984daccaf6ded089035b5fdc132", size = 2950909, upload-time = "2026-06-11T17:59:56.05Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/8d/f44198a859b13f519b331906d08079c7c281f8267d718082a3ca858c93ee/arro3_core-0.8.1-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d79b12a83403b1a100731587d33f09a818a4ff93472f84fcdf3a38d3934dfc5e", size = 3403497, upload-time = "2026-06-11T17:59:57.651Z" },
+    { url = "https://files.pythonhosted.org/packages/88/6d/431d2ef9942a30599db4a86cf13b7f1de92d340717438074a25659d382f4/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ff2da3c888ec504291deafd6965f2364f14f1fb63977f202f2f7680f10909f6", size = 3130901, upload-time = "2026-06-11T17:59:59.262Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/9e/458e8596c675fb88075137ab21d4c4a2f9e585101a97d77e910839862168/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0bbcfb60ddf9b571387b4c0aaf78171d2ee7e0d335c08514bc247685f01bc086", size = 3550071, upload-time = "2026-06-11T18:00:00.808Z" },
+    { url = "https://files.pythonhosted.org/packages/93/cd/8fa1f4a49e707ea364022620ca5033ba69b6396215e0bae1fb9fa9efcfe0/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:44f6256cae47d369fe9076e9fa61f5c2899447b76ca5bda641668ad07cf26bf6", size = 3508957, upload-time = "2026-06-11T18:00:02.341Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/02/0126ff3b2ff48187315a9782280c0f6412c066559e22bf206ceec2791299/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b2517ea3fb7cb15a41e2d0e3496d91afe4703eca86a88072f917bef044c2b8ee", size = 3408923, upload-time = "2026-06-11T18:00:03.97Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/fd/a19cb50480a769b1cc3f347efa1a808e13b853bb3b1d94b289677115fe92/arro3_core-0.8.1-cp311-abi3-win_amd64.whl", hash = "sha256:6a96df94a4538ab9acf01873eecf35161ad0c54ab79134247e69a42cd66515dc", size = 3368045, upload-time = "2026-06-11T18:00:05.578Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/86/1ac6eed229482b1ef6ac6f3211c6760012aaaa026b0a385a18e42ec0ff74/arro3_core-0.8.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:71165a75a7d22c16226085ed2255afe84741369cd35eb01cc153c4687e85b49f", size = 1724964, upload-time = "2026-06-11T18:00:07.653Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/32/c2e4a65b3b4f7e00552d280e9c2eaaca59c19466b0f95e4e08d6a020ab2d/arro3_core-0.8.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dc81e079d182dfc2ba6bace815608dbab0158809b9268c4eb59816685f7871fb", size = 3071070, upload-time = "2026-06-11T18:00:09.12Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/32/56016de27757bcc0d66999ae2fec624eaa27bd097ee69ac7a5a0d52d407b/arro3_core-0.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7695e9fca1e2c0064571bea65d9b7909a957ed9be4c81718d19356e578818649", size = 2794497, upload-time = "2026-06-11T18:00:10.872Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/6e/a7aac0e87d6d63672d4ebb7e8466b276fde2a6413579008fadd1e8e919c5/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad0f507dfafb1e8a8c15e8a902bccdd9a9c2cf2102fc39ad35c22e23ca76a65e", size = 3272090, upload-time = "2026-06-11T18:00:12.561Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/fd/8b96e57eea8ebf5fa4e2556b06c84a29ea25ded924f56d300807ff377a94/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fc91a59cb27c7660134b9ed0c067d53c0f5bb1722f16f2cf7d4c446ce1086c4", size = 3399753, upload-time = "2026-06-11T18:00:13.992Z" },
+    { url = "https://files.pythonhosted.org/packages/80/8b/76f1385ecd0448fcc60902a4474f3f00bd2197fe6d7fa4434bb6a0b3dee9/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71bc86880256cca5f22802ca444a92b3e9edbb5c9e5b74df1cbfcfe2ca788097", size = 3468912, upload-time = "2026-06-11T18:00:15.502Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/15/e1c3aefee23d926be9f75f125d6d94f189f28826aa01713498f1de60b080/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94528515f59146b5b4421e468bc269f682754ff082434cccf0ecbfb0d74d0b38", size = 3193338, upload-time = "2026-06-11T18:00:17.064Z" },
+    { url = "https://files.pythonhosted.org/packages/de/3d/bfa3963bd09941f69d810a387bbea61deb8a091101c1765216266ec61394/arro3_core-0.8.1-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:e348fa5349e6655df4b1ea281ab71e0794c749ba842da54248eb942201f93fb8", size = 2950166, upload-time = "2026-06-11T18:00:18.607Z" },
+    { url = "https://files.pythonhosted.org/packages/21/07/67bdc66582294d5914b8f7462907b1e44377ccb77a0f7e7b7760c9f81fd1/arro3_core-0.8.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d97170d56690655699977f4535120f170c5047813341c2b0264abc3af00625dc", size = 3401097, upload-time = "2026-06-11T18:00:20.135Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/a3/ab0d46e8fe2337f0ce1789949c5078e63d532fd628af4f254702e1359989/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6f71fdd46e1a86bc1db0733361130e51447d2d3c7d5fca13937381e2ba085dca", size = 3128454, upload-time = "2026-06-11T18:00:21.598Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/6e/e90cea2555aaa429c16173cbc121b41fb6a656e70ae9f21edbe3f9e73eb0/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3e2c2e4c6b36a0f493f07222eadf7c628155f4a306c650a14c4d6d343870043c", size = 3550294, upload-time = "2026-06-11T18:00:24.344Z" },
+    { url = "https://files.pythonhosted.org/packages/15/38/f3db72e411fbaea83844d19b1965abbdf3dd387083f99a715d3b3752b461/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90d31db38bc7cbbe6061d23ae11b3e830b55ce60ba6f0550c70915bab4cafe41", size = 3507806, upload-time = "2026-06-11T18:00:26.143Z" },
+    { url = "https://files.pythonhosted.org/packages/47/73/5d011dad78ac58aaa61e9cc65d349b879afec6781c5aa78f80c839fe359f/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9d28d87625d6c743fee1d2e20252531db6cff6c5218b89055a9f7492af99c14c", size = 3408910, upload-time = "2026-06-11T18:00:27.843Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/43/be86ea01c2d53d32376602cb2efa9a9b7a25d98e14ad4b5bf0d7da8b0564/arro3_core-0.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b394218ad71ce266088e0e65bfad0542556adf0eb93197b66afaa2009358662f", size = 3356174, upload-time = "2026-06-11T18:00:29.395Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/0f/2f7b5b458fd38ecf9277fd59919e4c90047795d1abdfd47f0081c48c28fa/arro3_core-0.8.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:a7c227982c0fd762271d550a6242e54ff7d2aa7aff73c918e127042cf9601ec0", size = 1710676, upload-time = "2026-06-11T18:00:30.947Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/9e/b64c9325173b7d87030955007af55348b866a1c02545382261d8966d8bb1/arro3_core-0.8.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5861781e6363db0707e1a4ba5166025a1113d921ccff870a0a097f27ab2dece1", size = 3072327, upload-time = "2026-06-11T18:00:32.313Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/bf/2c58a549b2409439fcac4ce2abe31ef060d05114a1e5412ce3088e2a2cde/arro3_core-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8c1d9ff8b2848bf48bbcad962271b2adeda922457cf23591464c12af9b0ddbf6", size = 2794872, upload-time = "2026-06-11T18:00:33.92Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/21/dee8d1c9309820783fe30ad29149508638743cdc61d2851f0132b370ac49/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:246057dc9d283a283cb05b228f710e1ada3b4a8c483e519b0ec8b0d8499204db", size = 3272432, upload-time = "2026-06-11T18:00:35.563Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/b2/dbc6a3d5ece2cb11c3041821c8322df500b8c8030534d9f791b66b29c090/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d55178ad56c637f278286370532e58256eff48f71d3c36a4e5f8652eb7300b3", size = 3400822, upload-time = "2026-06-11T18:00:37.129Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/44/0ca0e96eebc7f96af4ad8b78c545a9477e8db271aa090043bb8f797e7f58/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a917195d7553a188bbb9fa815d430ff70ee53357fcc6fdb395d0e4436a66243", size = 3469220, upload-time = "2026-06-11T18:00:39.375Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/70/3e9c63e9e4499d373304ceb1315afc8dd326395dd0692ccdebcfec703dab/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2435dc4057d604650a34d195e6977ef40b26af0b9dea8f1b842a924a270de2fc", size = 3193813, upload-time = "2026-06-11T18:00:41.281Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/a8/48842a836d7fc2bdac16ecaf4e60c9fd98f46d1e9c80f45b85b071ccaffb/arro3_core-0.8.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c8b90ad6fbdd3507c20bdbc2b23c88929a3f7c49a8a43697f316e084b5664289", size = 2950818, upload-time = "2026-06-11T18:00:42.987Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/46/635ca59c20b972f0575b9fd1b7debebd94bcba1d6fd68271bfff65d3ae91/arro3_core-0.8.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8918dd0fcf5c3650332678ddbfb7c13ce1f0534f9669ab6839c38069885318cf", size = 3401202, upload-time = "2026-06-11T18:00:44.654Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/03/020d032cf4937cca0652f434613d327f8f11e10b4a3fbd4ccf48846c3158/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cc8c7c9b81cad2b4eaaeb29da25854bddb5bb373a278f1f1046b277fa7f2ee6", size = 3129352, upload-time = "2026-06-11T18:00:46.322Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/55/ce1c840af64e8c5d4f8b684a4c00c5f6d260659c3d2f732c6074eb5597fe/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2aa0bda5251e93c67318aa8315e563b02fe39d5577b72654c335d69fce802d03", size = 3550475, upload-time = "2026-06-11T18:00:48.021Z" },
+    { url = "https://files.pythonhosted.org/packages/19/79/fa8f1a53cf10fe33029619d14f06525e08ea35c1bcdce42157b230098e1b/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:cd462e7ef7735c341c27f754756b244d90a9485818c5ea090fa46009bd8ae252", size = 3507877, upload-time = "2026-06-11T18:00:49.798Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/e5/bf8349b2e5613e6c0caf2f4f0758d61dc273649c2bdcfc96978ab9bc1686/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:480c33d62d66adf92fd731e77728f81b7bd3fcde4e2eb313f5616b5097b1875f", size = 3409360, upload-time = "2026-06-11T18:00:51.393Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/0d/e13efbbc448bc817f8343bf12606d22f83011682c821ccd32dda304816be/arro3_core-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:51bea8dc0bc0230b6af8d1b258cf2a7e4f69770ed99062592ce1b43b17732727", size = 3355557, upload-time = "2026-06-11T18:00:52.922Z" },
+    { url = "https://files.pythonhosted.org/packages/de/bc/ab3081cc65cd38a5ceda6dd112ed9d70a481e2810c8f92d37441edab6a8a/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d70c68665525744154dc1f65b5038fe97600a5204e8a875b3fe7675455363024", size = 3076025, upload-time = "2026-06-11T18:01:16.781Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/5b/b278236c7425f36fe1d25087964ca143d2a11f99b69b19f3c050cf3a0f4b/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b89c867f65852de7fa0586a4883195a2cca1ee3a7bca8c95c309a02ce6be15b5", size = 2797246, upload-time = "2026-06-11T18:01:18.648Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/a5/a3a96bef25a7d0d7fd49ea3762d901a17d908fda774c868abe9aa065e0f2/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66d685f7c53137798e05322a17dabc09445a818c9c8959d21e9fb3e084371f51", size = 3269959, upload-time = "2026-06-11T18:01:20.317Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/2f/4da148dea5890ae24eade70d3f9dfc93d191f501bf425d906c21dee89510/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a7563856e609dd24be03efde72edd5b1c16dd1fb3d67fd84970df6d9ca36b96", size = 3400877, upload-time = "2026-06-11T18:01:22.112Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/9a/2ed015d4a03c131b9e3da322ee8681c5921320a90218865a711d7fe39768/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7379e14f8f49e99d53e364c26533293d2136920a3cf818138718a9a9bc114d0", size = 3466119, upload-time = "2026-06-11T18:01:23.909Z" },
+    { url = "https://files.pythonhosted.org/packages/25/a8/154478c9b3093602eaec8a5d71b657205177f0ac9fd9da2eef9ddfb42fe0/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af88a17346e9e7897b6d1bc165f686b87ea928b4c7e89666156b2281f0236cee", size = 3190211, upload-time = "2026-06-11T18:01:25.786Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/65/60c6323bbaffd2399f03336a90d7c02450a5b9120348808eb0260651de56/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:1e9e04cbbba877f31073146798b328537b63675f65d0b7e2154754d5203260ed", size = 2950345, upload-time = "2026-06-11T18:01:27.423Z" },
+    { url = "https://files.pythonhosted.org/packages/21/81/3a0c786ec86a8c3aee946e4588b188f08a3ccbba71c4e0db144a0151b56c/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:35c46c49442853f8a1b09d37ceabe4ce31e2a276d4a73c407ba3a25fcf1b9221", size = 3398153, upload-time = "2026-06-11T18:01:29.036Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/52/e230d0af9881056e527f91e58c518b85000e9ae49f8706c3d0e027cbadd4/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4e4af152b4ccf7694d634573db906e1a9b28f5beda84bac5adeb60f7e98262a8", size = 3128123, upload-time = "2026-06-11T18:01:30.733Z" },
+    { url = "https://files.pythonhosted.org/packages/62/79/9bc36279f1ef2754fde18e1c3c282b2cabe6093b7ca8bd8c0c6c735685ed/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:7bf8cb1053cb51529437544eedbc6643819912eec15c0491d7304d6bad0fb151", size = 3547345, upload-time = "2026-06-11T18:01:32.603Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/33/cee8670131009d3a23e140dd0a6cd454111dac54a9a2e83567a6f972ad46/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:212dfde17bd26193754aaee6024f7e99624163e0d44966745e8752af79c4c454", size = 3504672, upload-time = "2026-06-11T18:01:34.207Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/5f/daa7a8af9ad53eebbdb818078d5034525663f298ec9bfdb1933cfd0cc694/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:81426cb122d6bfccacd2e0ae5df8b847dc3c3e07b5f336547cb18f873d186f54", size = 3404906, upload-time = "2026-06-11T18:01:36.097Z" },
 ]
 
 [[package]]
@@ -272,11 +279,11 @@ wheels = [
 
 [[package]]
 name = "attrs"
-version = "25.3.0"
+version = "26.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" },
+    { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" },
 ]
 
 [[package]]
@@ -295,89 +302,133 @@ wheels = [
 
 [[package]]
 name = "botocore"
-version = "1.40.43"
+version = "1.40.76"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jmespath" },
     { name = "python-dateutil" },
     { name = "urllib3" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164, upload-time = "2025-10-01T19:38:16.06Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/07/eb/50e2d280589a3c20c3b649bb66262d2b53a25c03262e4cc492048ac7540a/botocore-1.40.76.tar.gz", hash = "sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc", size = 14494001, upload-time = "2025-11-18T20:22:59.131Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/79/46/2eb4802e15e38befbea6cab7dafa1ab796722ab6f0833991c2a05e9f8ef0/botocore-1.40.43-py3-none-any.whl", hash = "sha256:1639f38999fc0cf42c92c5c83c5fbe189a4857a86f55b842be868e3283c6d3bb", size = 14057986, upload-time = "2025-10-01T19:38:13.714Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" },
 ]
 
 [[package]]
 name = "certifi"
-version = "2025.8.3"
+version = "2026.6.17"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
 ]
 
 [[package]]
 name = "charset-normalizer"
-version = "3.4.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" },
-    { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" },
-    { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" },
-    { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" },
-    { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" },
-    { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" },
-    { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" },
-    { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" },
-    { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" },
-    { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" },
-    { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" },
-    { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" },
-    { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" },
-    { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" },
-    { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" },
-    { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" },
-    { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" },
-    { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" },
-    { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" },
-    { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" },
-    { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" },
-    { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" },
-    { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" },
-    { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" },
-    { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" },
-    { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" },
-    { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" },
-    { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" },
+version = "3.4.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a", size = 322240, upload-time = "2026-07-07T14:32:36.236Z" },
+    { url = "https://files.pythonhosted.org/packages/de/d1/b4319dc3229d8272fba305e206fc0a148e2de8d4087917ce62ae6382f359/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616", size = 216475, upload-time = "2026-07-07T14:32:38.142Z" },
+    { url = "https://files.pythonhosted.org/packages/80/33/6c99c1b3e6b8bf730e1bc809b9a2608f224145069114c479a2e9e1494346/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209", size = 238670, upload-time = "2026-07-07T14:32:39.658Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/f4/ffbb83546e1f198ecc70ecd372b65cf2b50f9068b380abd67640f17a8e18/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99", size = 233476, upload-time = "2026-07-07T14:32:41.155Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8", size = 223817, upload-time = "2026-07-07T14:32:42.592Z" },
+    { url = "https://files.pythonhosted.org/packages/36/31/a276bb2e66243072a3fd06fdcab9cbb61a305b02143d70d2bda21d888fa8/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b", size = 207974, upload-time = "2026-07-07T14:32:44.258Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/be/7ee4453d7e88dfbc4104ccd34900b9f2c7c17dac22881865fe0e82424a25/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2", size = 221655, upload-time = "2026-07-07T14:32:45.64Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/85/181c652953eb5276d198f375b1dd641047392050098100a3a02d6534f657/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9", size = 219229, upload-time = "2026-07-07T14:32:47.376Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/e7/aaf6da33fc9f4691cda8f7efbc9f69179d3d39ec8a4799baf273ee1d8db0/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15", size = 209704, upload-time = "2026-07-07T14:32:48.855Z" },
+    { url = "https://files.pythonhosted.org/packages/63/01/f2fb3bd3a73be48b173ee0c6aa8d2497af97d5663a8c4c4b491de4c62f7a/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d", size = 226243, upload-time = "2026-07-07T14:32:50.239Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/02/c57a22739fe05246b0b5783b3bfb6afaac4eebb46f3ececdfb2f048f780e/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381", size = 150935, upload-time = "2026-07-07T14:32:51.676Z" },
+    { url = "https://files.pythonhosted.org/packages/37/8d/ca39a7559a4797505530d084fd3a49a2c959efbbbff146302fb7be4e3b35/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee", size = 162314, upload-time = "2026-07-07T14:32:53.193Z" },
+    { url = "https://files.pythonhosted.org/packages/01/da/a44bd7a13d426e69e4894557106cd58669097bfad4a8681123b618fbfc5d/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419", size = 153075, upload-time = "2026-07-07T14:32:54.554Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" },
+    { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" },
+    { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" },
+    { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" },
+    { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" },
+    { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" },
+    { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" },
+    { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" },
+    { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" },
+    { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" },
+    { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" },
+    { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" },
+    { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" },
+    { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" },
+    { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" },
+    { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" },
+    { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" },
+    { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" },
+    { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" },
+    { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" },
+    { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" },
+    { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" },
+    { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" },
+    { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" },
+    { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" },
+]
+
+[[package]]
+name = "click"
+version = "8.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" },
+]
+
+[[package]]
+name = "cloudpickle"
+version = "3.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" },
 ]
 
 [[package]]
@@ -413,81 +464,98 @@ wheels = [
 
 [[package]]
 name = "cuda-pathfinder"
-version = "1.5.5"
+version = "1.5.6"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0", size = 52972, upload-time = "2026-06-30T00:58:04.34Z" },
 ]
 
 [[package]]
 name = "cuda-toolkit"
-version = "13.0.2"
+version = "13.0.3.0"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" },
 ]
 
 [package.optional-dependencies]
+cublas = [
+    { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
 cudart = [
-    { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 cufft = [
-    { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 cufile = [
-    { name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
+    { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
 ]
 cupti = [
-    { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 curand = [
-    { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 cusolver = [
-    { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 cusparse = [
-    { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 nvjitlink = [
-    { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 nvrtc = [
-    { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 nvtx = [
-    { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
 ]
 
 [[package]]
 name = "datafusion"
-version = "53.0.0"
+version = "54.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "cloudpickle" },
     { name = "pyarrow" },
     { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" },
-    { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" },
+    { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" },
+    { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" },
+    { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" },
+    { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" },
+    { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" },
+    { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" },
 ]
 
 [[package]]
 name = "datasets"
-version = "4.1.1"
+version = "4.4.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "dill" },
     { name = "filelock" },
     { name = "fsspec", extra = ["http"] },
+    { name = "httpx" },
     { name = "huggingface-hub" },
     { name = "multiprocess" },
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
+    { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "packaging" },
     { name = "pandas" },
     { name = "pyarrow" },
@@ -496,9 +564,21 @@ dependencies = [
     { name = "tqdm" },
     { name = "xxhash" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324, upload-time = "2025-09-18T13:14:27.108Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/57/13/f05a80bbbac5f62e492e5e463ec59a4479647ef9c376b1fdfaa4d3ed01cc/datasets-4.4.0.tar.gz", hash = "sha256:0430d39b9f13b53c37afb80c23c7e5d8c6ceccc014c14a14d15fa2b4e8688d2a", size = 585143, upload-time = "2025-11-04T10:36:22.73Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1e/31/d552336985f747b19f0a852d98ca7a2ef4727ba956b38041cfbda08dde0a/datasets-4.4.0-py3-none-any.whl", hash = "sha256:b7e6d1d48c2e1d3a95d6b378e8fc3d7ab29f24f14ddf505a8d417dd09c692f19", size = 511463, upload-time = "2025-11-04T10:36:20.062Z" },
+]
+
+[[package]]
+name = "deprecated"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623, upload-time = "2025-09-18T13:14:25.111Z" },
+    { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" },
 ]
 
 [[package]]
@@ -512,167 +592,195 @@ wheels = [
 
 [[package]]
 name = "duckdb"
-version = "1.4.0"
+version = "1.5.5"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175, upload-time = "2025-09-16T10:22:41.509Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0f/4a/b2e17dbe2953481b084f355f162ed319a67ef760e28794c6870058583aec/duckdb-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e24e981a6c87e299201694b9bb24fff0beb04ccad399fca6f13072a59814488f", size = 31293005, upload-time = "2025-09-16T10:21:28.296Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/89/e34ed03cce7e35b83c1f056126aa4e8e8097eb93e7324463020f85d5cbfa/duckdb-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db500ef2c8cb7dc1ca078740ecf1dceaa20d3f5dc5bce269be45d5cff4170c0f", size = 17288207, upload-time = "2025-09-16T10:21:31.129Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/17/7ff24799ee98c4dbb177c3ec6c93e38e9513828785c31757c727b47ad71e/duckdb-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a65739b8a7106634e6e77d0e110fc5e057b88edc9df6cb1683d499a1e5aa3177", size = 14817523, upload-time = "2025-09-16T10:21:33.397Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/ab/7a482a76ff75212b5cf4f2172a802f2a59b4ab096416e5821aa62a305bc4/duckdb-1.4.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59f7be24862adb803a1ddfc9c3b8cb09e6005bca0c9c6f7c631a1da1c3aa0c", size = 18410654, upload-time = "2025-09-16T10:21:35.864Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/f6/a235233b973652b31448b6d600604620d02fc552b90ab94ca7f645fd5ac0/duckdb-1.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d052a87e9edf4eb3bab0b7a6ac995676018c6083b8049421628dfa3b983a2d4", size = 20399121, upload-time = "2025-09-16T10:21:38.524Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/cf/63fedb74d00d7c4e19ffc73a1d8d98ee8d3d6498cf2865509c104aa8e799/duckdb-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:0329b81e587f745b2fc6f3a488ea3188b0f029c3b5feef43792a25eaac84ac01", size = 12283288, upload-time = "2025-09-16T10:21:40.732Z" },
-    { url = "https://files.pythonhosted.org/packages/60/e9/b29cc5bceac52e049b20d613551a2171a092df07f26d4315f3f9651c80d4/duckdb-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6505fed1ccae8df9f574e744c48fa32ee2feaeebe5346c2daf4d4d10a8dac5aa", size = 31290878, upload-time = "2025-09-16T10:21:43.256Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/68/d88a15dba48bf6a4b33f1be5097ef45c83f7b9e97c854cc638a85bb07d70/duckdb-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36974a04b29c74ac2143457e95420a7422016d050e28573060b89a90b9cf2b57", size = 17288823, upload-time = "2025-09-16T10:21:45.716Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/7e/e3d2101dc6bbd60f2b3c1d748351ff541fc8c48790ac1218c0199cb930f6/duckdb-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90484b896e5059f145d1facfabea38e22c54a2dcc2bd62dd6c290423f0aee258", size = 14819684, upload-time = "2025-09-16T10:21:48.117Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/bb/4ec8e4d03cb5b77d75b9ee0057c2c714cffaa9bda1e55ffec833458af0a3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a969d624b385853b31a43b0a23089683297da2f14846243921c6dbec8382d659", size = 18410075, upload-time = "2025-09-16T10:21:50.517Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/21/e896616d892d50dc1e0c142428e9359b483d4dd6e339231d822e57834ad3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5935644f96a75e9f6f3c3eeb3da14cdcaf7bad14d1199c08439103decb29466a", size = 20402984, upload-time = "2025-09-16T10:21:52.808Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/c0/b5eb9497e4a9167d23fbad745969eaa36e28d346648e17565471892d1b33/duckdb-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:300aa0e963af97969c38440877fffd576fc1f49c1f5914789a9d01f2fe7def91", size = 12282971, upload-time = "2025-09-16T10:21:55.314Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425, upload-time = "2025-09-16T10:21:57.83Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727, upload-time = "2025-09-16T10:22:02.242Z" },
-    { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879, upload-time = "2025-09-16T10:22:05.162Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856, upload-time = "2025-09-16T10:22:07.919Z" },
-    { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154, upload-time = "2025-09-16T10:22:10.216Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108, upload-time = "2025-09-16T10:22:12.668Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/64/ee22b2b8572746e1523143b9f28d606575782e0204de5020656a1d15dd14/duckdb-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d94d010a09b1a62d9021a2a71cf266188750f3c9b1912ccd6afe104a6ce8010", size = 31307662, upload-time = "2025-09-16T10:22:14.9Z" },
-    { url = "https://files.pythonhosted.org/packages/76/2e/4241cd00046ca6b781bd1d9002e8223af061e85d1cc21830aa63e7a7db7c/duckdb-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c61756fa8b3374627e5fa964b8e0d5b58e364dce59b87dba7fb7bc6ede196b26", size = 17295617, upload-time = "2025-09-16T10:22:17.239Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/98/5ab136bc7b12ac18580350a220db7c00606be9eac2d89de259cce733f64c/duckdb-1.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e70d7d9881ea2c0836695de70ea68c970e18a2856ba3d6502e276c85bd414ae7", size = 14826727, upload-time = "2025-09-16T10:22:19.415Z" },
-    { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289, upload-time = "2025-09-16T10:22:21.564Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547, upload-time = "2025-09-16T10:22:23.759Z" },
-    { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467, upload-time = "2025-09-16T10:22:25.923Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/d4/298acf9331a80b3ce6ac64dd940e7e13f4058fb69d18914445f02e3c7bfe/duckdb-1.5.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b805507f88171b428b21c966c30e9a3d54e30b24528918a44ed0032542bc26f", size = 32702934, upload-time = "2026-07-22T10:53:19.069Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/90/c489fb63d64b2e7ee109ce8460bdede003a0f256e5b41a03a2a1c4764058/duckdb-1.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b08e19cc856220d8a26fa62abc2264b349aff67255e9373c6a3f607addd56dc6", size = 17343604, upload-time = "2026-07-22T10:53:22.767Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/27/effa80a15b1f0c61c235622f797868485359e8c9ad6a8e358e7a0c479151/duckdb-1.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a17e6a922e42a5c06ed2353fe78c5dff2610f6632d603836f9606ad0bf754079", size = 15488179, upload-time = "2026-07-22T10:53:25.945Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/07/21212345c8d24ba62dceaa20be3b21f5c46f1510b1b42ce93bb058afe0c4/duckdb-1.5.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc38922c365c37720149f90d90b1e9823eb82dad6830855b5f87537fa6fc0c", size = 19367323, upload-time = "2026-07-22T10:53:30.23Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/d0/10371ae875fb4b5ef61bb892743b4b2e90c512b371fdf29317deb744857d/duckdb-1.5.5-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e238060db5ca59879882a6e9b015e2c65d5c64ddf281ba1d7a9a2033764152cf", size = 21476568, upload-time = "2026-07-22T10:53:33.486Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/35/09568ce617dd7bc0757b3d7b6a981660b9e4f0b7594de8ed776755eae740/duckdb-1.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:4acc72798ba1885a9c17d1242903d2cd502f13b1271c7677f7cab25d8578eceb", size = 13156129, upload-time = "2026-07-22T10:53:37.55Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" },
+    { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" },
+    { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" },
+    { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" },
+    { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" },
+    { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" },
+    { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" },
+    { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" },
+    { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" },
 ]
 
 [[package]]
 name = "exceptiongroup"
-version = "1.3.0"
+version = "1.3.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
 ]
 
 [[package]]
 name = "filelock"
-version = "3.19.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
-]
-
-[[package]]
-name = "flatbuffers"
-version = "25.9.23"
+version = "3.29.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067, upload-time = "2025-09-24T05:25:30.106Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/35/94/00f2059e4835eace3ae8fde680b932c496f8ec7bdc99168dfa53fb2e6b79/filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d", size = 71521, upload-time = "2026-07-08T05:46:58.716Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869, upload-time = "2025-09-24T05:25:28.912Z" },
+    { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036, upload-time = "2026-07-08T05:46:57.53Z" },
 ]
 
 [[package]]
 name = "frozenlist"
-version = "1.7.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304, upload-time = "2025-06-09T22:59:46.226Z" },
-    { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735, upload-time = "2025-06-09T22:59:48.133Z" },
-    { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775, upload-time = "2025-06-09T22:59:49.564Z" },
-    { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644, upload-time = "2025-06-09T22:59:51.35Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125, upload-time = "2025-06-09T22:59:52.884Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455, upload-time = "2025-06-09T22:59:54.74Z" },
-    { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339, upload-time = "2025-06-09T22:59:56.187Z" },
-    { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969, upload-time = "2025-06-09T22:59:57.604Z" },
-    { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862, upload-time = "2025-06-09T22:59:59.498Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492, upload-time = "2025-06-09T23:00:01.026Z" },
-    { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250, upload-time = "2025-06-09T23:00:03.401Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720, upload-time = "2025-06-09T23:00:05.282Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585, upload-time = "2025-06-09T23:00:07.962Z" },
-    { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248, upload-time = "2025-06-09T23:00:09.428Z" },
-    { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621, upload-time = "2025-06-09T23:00:11.32Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578, upload-time = "2025-06-09T23:00:13.526Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830, upload-time = "2025-06-09T23:00:14.98Z" },
-    { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251, upload-time = "2025-06-09T23:00:16.279Z" },
-    { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183, upload-time = "2025-06-09T23:00:17.698Z" },
-    { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107, upload-time = "2025-06-09T23:00:18.952Z" },
-    { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333, upload-time = "2025-06-09T23:00:20.275Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724, upload-time = "2025-06-09T23:00:21.705Z" },
-    { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842, upload-time = "2025-06-09T23:00:23.148Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767, upload-time = "2025-06-09T23:00:25.103Z" },
-    { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130, upload-time = "2025-06-09T23:00:27.061Z" },
-    { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301, upload-time = "2025-06-09T23:00:29.02Z" },
-    { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606, upload-time = "2025-06-09T23:00:30.514Z" },
-    { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372, upload-time = "2025-06-09T23:00:31.966Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860, upload-time = "2025-06-09T23:00:33.375Z" },
-    { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893, upload-time = "2025-06-09T23:00:35.002Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323, upload-time = "2025-06-09T23:00:36.468Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149, upload-time = "2025-06-09T23:00:37.963Z" },
-    { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565, upload-time = "2025-06-09T23:00:39.753Z" },
-    { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019, upload-time = "2025-06-09T23:00:40.988Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" },
-    { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" },
-    { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" },
-    { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" },
-    { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" },
-    { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" },
-    { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" },
-    { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" },
-    { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" },
-    { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" },
-    { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" },
-    { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" },
-    { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" },
-    { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" },
-    { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" },
-    { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" },
-    { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" },
-    { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" },
-    { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" },
-    { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" },
-    { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" },
-    { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" },
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/83/4a/557715d5047da48d54e659203b9335be7bfaafda2c3f627b7c47e0b3aaf3/frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011", size = 86230, upload-time = "2025-10-06T05:35:23.699Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/fb/c85f9fed3ea8fe8740e5b46a59cc141c23b842eca617da8876cfce5f760e/frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565", size = 49621, upload-time = "2025-10-06T05:35:25.341Z" },
+    { url = "https://files.pythonhosted.org/packages/63/70/26ca3f06aace16f2352796b08704338d74b6d1a24ca38f2771afbb7ed915/frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad", size = 49889, upload-time = "2025-10-06T05:35:26.797Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2", size = 219464, upload-time = "2025-10-06T05:35:28.254Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/83/4d587dccbfca74cb8b810472392ad62bfa100bf8108c7223eb4c4fa2f7b3/frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186", size = 221649, upload-time = "2025-10-06T05:35:29.454Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/c6/fd3b9cd046ec5fff9dab66831083bc2077006a874a2d3d9247dea93ddf7e/frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e", size = 219188, upload-time = "2025-10-06T05:35:30.951Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/80/6693f55eb2e085fc8afb28cf611448fb5b90e98e068fa1d1b8d8e66e5c7d/frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450", size = 231748, upload-time = "2025-10-06T05:35:32.101Z" },
+    { url = "https://files.pythonhosted.org/packages/97/d6/e9459f7c5183854abd989ba384fe0cc1a0fb795a83c033f0571ec5933ca4/frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef", size = 236351, upload-time = "2025-10-06T05:35:33.834Z" },
+    { url = "https://files.pythonhosted.org/packages/97/92/24e97474b65c0262e9ecd076e826bfd1d3074adcc165a256e42e7b8a7249/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4", size = 218767, upload-time = "2025-10-06T05:35:35.205Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/bf/dc394a097508f15abff383c5108cb8ad880d1f64a725ed3b90d5c2fbf0bb/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff", size = 235887, upload-time = "2025-10-06T05:35:36.354Z" },
+    { url = "https://files.pythonhosted.org/packages/40/90/25b201b9c015dbc999a5baf475a257010471a1fa8c200c843fd4abbee725/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c", size = 228785, upload-time = "2025-10-06T05:35:37.949Z" },
+    { url = "https://files.pythonhosted.org/packages/84/f4/b5bc148df03082f05d2dd30c089e269acdbe251ac9a9cf4e727b2dbb8a3d/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f", size = 230312, upload-time = "2025-10-06T05:35:39.178Z" },
+    { url = "https://files.pythonhosted.org/packages/db/4b/87e95b5d15097c302430e647136b7d7ab2398a702390cf4c8601975709e7/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7", size = 217650, upload-time = "2025-10-06T05:35:40.377Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/70/78a0315d1fea97120591a83e0acd644da638c872f142fd72a6cebee825f3/frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a", size = 39659, upload-time = "2025-10-06T05:35:41.863Z" },
+    { url = "https://files.pythonhosted.org/packages/66/aa/3f04523fb189a00e147e60c5b2205126118f216b0aa908035c45336e27e4/frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6", size = 43837, upload-time = "2025-10-06T05:35:43.205Z" },
+    { url = "https://files.pythonhosted.org/packages/39/75/1135feecdd7c336938bd55b4dc3b0dfc46d85b9be12ef2628574b28de776/frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e", size = 39989, upload-time = "2025-10-06T05:35:44.596Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" },
+    { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" },
+    { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" },
+    { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" },
+    { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" },
+    { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" },
+    { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" },
+    { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" },
+    { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" },
+    { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" },
+    { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" },
+    { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" },
+    { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" },
+    { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" },
+    { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" },
+    { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" },
+    { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" },
+    { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" },
+    { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" },
+    { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" },
+    { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" },
+    { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" },
+    { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" },
+    { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" },
+    { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" },
+    { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" },
+    { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" },
+    { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" },
 ]
 
 [[package]]
 name = "fsspec"
-version = "2025.9.0"
+version = "2025.10.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" },
 ]
 
 [package.optional-dependencies]
@@ -680,15 +788,6 @@ http = [
     { name = "aiohttp" },
 ]
 
-[[package]]
-name = "gast"
-version = "0.6.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708, upload-time = "2024-06-27T20:31:49.527Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173, upload-time = "2024-07-09T13:15:15.615Z" },
-]
-
 [[package]]
 name = "geoarrow-rust-core"
 version = "0.6.3"
@@ -806,133 +905,122 @@ wheels = [
 ]
 
 [[package]]
-name = "google-pasta"
-version = "0.2.0"
+name = "h11"
+version = "0.16.0"
 source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "six" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" },
+    { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]
 
 [[package]]
-name = "grpcio"
-version = "1.75.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534, upload-time = "2025-09-26T09:00:52.225Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953, upload-time = "2025-09-26T09:01:03.699Z" },
-    { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785, upload-time = "2025-09-26T09:01:07.504Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708, upload-time = "2025-09-26T09:01:11.028Z" },
-    { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912, upload-time = "2025-09-26T09:01:14.393Z" },
-    { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497, upload-time = "2025-09-26T09:01:17.759Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284, upload-time = "2025-09-26T09:01:20.896Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761, upload-time = "2025-09-26T09:01:28.528Z" },
-    { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084, upload-time = "2025-09-26T09:01:34.596Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403, upload-time = "2025-09-26T09:01:36.736Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166, upload-time = "2025-09-26T09:01:39.474Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828, upload-time = "2025-09-26T09:01:41.782Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421, upload-time = "2025-09-26T09:01:45.835Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290, upload-time = "2025-09-26T09:01:49.28Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" },
-    { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" },
-    { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779, upload-time = "2025-09-26T09:02:19.11Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838, upload-time = "2025-09-26T09:02:26.426Z" },
-    { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663, upload-time = "2025-09-26T09:02:28.724Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149, upload-time = "2025-09-26T09:02:30.971Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989, upload-time = "2025-09-26T09:02:33.233Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717, upload-time = "2025-09-26T09:02:36.011Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822, upload-time = "2025-09-26T09:02:38.26Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319, upload-time = "2025-09-26T09:02:44.742Z" },
-    { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706, upload-time = "2025-09-26T09:02:50.4Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501, upload-time = "2025-09-26T09:02:52.711Z" },
-    { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492, upload-time = "2025-09-26T09:02:55.542Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061, upload-time = "2025-09-26T09:02:58.261Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849, upload-time = "2025-09-26T09:03:00.548Z" },
-    { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478, upload-time = "2025-09-26T09:03:03.096Z" },
-]
-
-[[package]]
-name = "h5py"
-version = "3.14.0"
+name = "hf-xet"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/64/ee/dd9ba7beae1005e54131b7d45263cc74c8a066d47d354e6d58ae9445a388/hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", size = 4069485, upload-time = "2026-06-08T23:02:13.193Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/bc/9cae6cfeb4e03070874e73e5c97c66eb90369d3206b6a2b1ef5f96520888/hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", size = 3838493, upload-time = "2026-06-08T23:02:15.282Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/b4/d5c01e0eb6d9f2ca2dacd84d0d1b71e6cfbb2ef3208c968528e010e9b3d7/hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", size = 4505658, upload-time = "2026-06-08T23:02:17.196Z" },
+    { url = "https://files.pythonhosted.org/packages/76/c5/29a7598c0c6383c523dc22186d577f4e04267a626cd95ae60f67c00bfe66/hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", size = 4292822, upload-time = "2026-06-08T23:02:18.608Z" },
+    { url = "https://files.pythonhosted.org/packages/04/9a/dceaf6ca69390126b86ea825fb354b93d01163199070b7bd849225de9468/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", size = 4491255, upload-time = "2026-06-08T23:02:20.124Z" },
+    { url = "https://files.pythonhosted.org/packages/48/a7/e5a7afaacf6c1791fdbeeac42951fb81c3d2bc482992b115dedcc86d963e/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", size = 4711062, upload-time = "2026-06-08T23:02:21.863Z" },
+    { url = "https://files.pythonhosted.org/packages/53/49/2802f8433c9742ce281bddc1e65c02c32268ca3098d66828b05e12e45ee2/hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", size = 4017205, upload-time = "2026-06-08T23:02:23.495Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/5a/50c71195b9fb883659f596e7252faf4c18c58e753a9013bdbf9bac5d2250/hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", size = 3845426, upload-time = "2026-06-08T23:02:25.124Z" },
+    { url = "https://files.pythonhosted.org/packages/05/24/5e0c28f80371c17d49fed004597d9d132cb75c1f6f53db2cb95f459d2312/hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", size = 4069676, upload-time = "2026-06-08T23:02:26.759Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/17/261ba565b6a4d960fb478f61fdf919c0be5824645aaf1c319eca660c1611/hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", size = 3838509, upload-time = "2026-06-08T23:02:28.573Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/44/7ffdc2e184b0d41fc0f683ba3936ef669ab63cf242cf36ef50e57d683668/hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", size = 4505881, upload-time = "2026-06-08T23:02:30.257Z" },
+    { url = "https://files.pythonhosted.org/packages/63/b6/788060d5aa4d5e671f1a31bf69624c314eb2d8babab3aa562f9e5d53444e/hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", size = 4292995, upload-time = "2026-06-08T23:02:31.993Z" },
+    { url = "https://files.pythonhosted.org/packages/22/93/c5540cbd6b55529b7dc42f6734e88cebee21aefbea34128b66229df56c57/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", size = 4491570, upload-time = "2026-06-08T23:02:33.86Z" },
+    { url = "https://files.pythonhosted.org/packages/03/f3/9d8ceab30f44f36c1679b1b8683054c71a0dadc787dbf07421891742d3ca/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", size = 4711565, upload-time = "2026-06-08T23:02:35.454Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/54/27ed9a5e2cc583b4df82f75a03a4df8dbf55f5a9fa1f47f1fadfb20dbeac/hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", size = 4017343, upload-time = "2026-06-08T23:02:37.14Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/12/ecb2fc8d45e767580e3a37faa97cb895608b614965567efb4f18cff67e27/hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", size = 3845716, upload-time = "2026-06-08T23:02:39.073Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" },
+    { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" },
+    { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" },
+    { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" },
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.9"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "certifi" },
+    { name = "h11" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" },
-    { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" },
-    { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" },
-    { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
 ]
 
 [[package]]
-name = "hf-xet"
-version = "1.1.10"
+name = "httpx"
+version = "0.28.1"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" }
+dependencies = [
+    { name = "anyio" },
+    { name = "certifi" },
+    { name = "httpcore" },
+    { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" },
-    { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" },
-    { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" },
-    { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
 ]
 
 [[package]]
 name = "huggingface-hub"
-version = "0.35.3"
+version = "1.23.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "click" },
     { name = "filelock" },
     { name = "fsspec" },
-    { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "httpx" },
     { name = "packaging" },
     { name = "pyyaml" },
-    { name = "requests" },
     { name = "tqdm" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1a/8f/999e4dda11c6187c78f090eac00895a47e11a0049308f07579bcb7aa3aa2/huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88", size = 919163, upload-time = "2026-07-09T14:49:32.315Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/ce/13b2ba57838b8db1e6bd033c1b21ce0b9f6153b87d4e4939f77074e41eb0/huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2", size = 770336, upload-time = "2026-07-09T14:49:30.597Z" },
 ]
 
 [[package]]
 name = "idna"
-version = "3.15"
+version = "3.18"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" },
+]
+
+[[package]]
+name = "importlib-metadata"
+version = "8.5.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "zipp" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" },
 ]
 
 [[package]]
 name = "iniconfig"
-version = "2.1.0"
+version = "2.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
 ]
 
 [[package]]
@@ -949,31 +1037,11 @@ wheels = [
 
 [[package]]
 name = "jmespath"
-version = "1.0.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
-]
-
-[[package]]
-name = "keras"
-version = "3.11.3"
+version = "1.1.0"
 source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "absl-py" },
-    { name = "h5py" },
-    { name = "ml-dtypes" },
-    { name = "namex" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "optree" },
-    { name = "packaging" },
-    { name = "rich" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906, upload-time = "2025-08-21T22:08:57.643Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438, upload-time = "2025-08-21T22:08:55.858Z" },
+    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
 ]
 
 [[package]]
@@ -1003,39 +1071,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" },
 ]
 
-[[package]]
-name = "libclang"
-version = "18.1.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" },
-]
-
-[[package]]
-name = "markdown"
-version = "3.9"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" },
-]
-
-[[package]]
-name = "markdown-it-py"
-version = "4.0.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "mdurl" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
-]
-
 [[package]]
 name = "markupsafe"
 version = "3.0.3"
@@ -1145,22 +1180,14 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6c/93/e32e79333f0902ba292b996f504f5f06be59587f7d02ab8d5ed1e3066445/maturin-1.13.3-py3-none-win_arm64.whl", hash = "sha256:2389fe92d017cea9d94e521fa0175314a4c52f79a1057b901fbc9f8686ef7d0b", size = 9706562, upload-time = "2026-05-11T07:43:31.743Z" },
 ]
 
-[[package]]
-name = "mdurl"
-version = "0.1.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
-]
-
 [[package]]
 name = "ml-dtypes"
 version = "0.5.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
+    { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316, upload-time = "2025-07-29T18:39:19.454Z" }
 wheels = [
@@ -1207,131 +1234,163 @@ wheels = [
 
 [[package]]
 name = "multidict"
-version = "6.6.4"
+version = "6.7.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/eb/6b/86f353088c1358e76fd30b0146947fddecee812703b604ee901e85cd2a80/multidict-6.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b8aa6f0bd8125ddd04a6593437bad6a7e70f300ff4180a531654aa2ab3f6d58f", size = 77054, upload-time = "2025-08-11T12:06:02.99Z" },
-    { url = "https://files.pythonhosted.org/packages/19/5d/c01dc3d3788bb877bd7f5753ea6eb23c1beeca8044902a8f5bfb54430f63/multidict-6.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9e5853bbd7264baca42ffc53391b490d65fe62849bf2c690fa3f6273dbcd0cb", size = 44914, upload-time = "2025-08-11T12:06:05.264Z" },
-    { url = "https://files.pythonhosted.org/packages/46/44/964dae19ea42f7d3e166474d8205f14bb811020e28bc423d46123ddda763/multidict-6.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0af5f9dee472371e36d6ae38bde009bd8ce65ac7335f55dcc240379d7bed1495", size = 44601, upload-time = "2025-08-11T12:06:06.627Z" },
-    { url = "https://files.pythonhosted.org/packages/31/20/0616348a1dfb36cb2ab33fc9521de1f27235a397bf3f59338e583afadd17/multidict-6.6.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:d24f351e4d759f5054b641c81e8291e5d122af0fca5c72454ff77f7cbe492de8", size = 224821, upload-time = "2025-08-11T12:06:08.06Z" },
-    { url = "https://files.pythonhosted.org/packages/14/26/5d8923c69c110ff51861af05bd27ca6783011b96725d59ccae6d9daeb627/multidict-6.6.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db6a3810eec08280a172a6cd541ff4a5f6a97b161d93ec94e6c4018917deb6b7", size = 242608, upload-time = "2025-08-11T12:06:09.697Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/cc/e2ad3ba9459aa34fa65cf1f82a5c4a820a2ce615aacfb5143b8817f76504/multidict-6.6.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a1b20a9d56b2d81e2ff52ecc0670d583eaabaa55f402e8d16dd062373dbbe796", size = 222324, upload-time = "2025-08-11T12:06:10.905Z" },
-    { url = "https://files.pythonhosted.org/packages/19/db/4ed0f65701afbc2cb0c140d2d02928bb0fe38dd044af76e58ad7c54fd21f/multidict-6.6.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c9854df0eaa610a23494c32a6f44a3a550fb398b6b51a56e8c6b9b3689578db", size = 253234, upload-time = "2025-08-11T12:06:12.658Z" },
-    { url = "https://files.pythonhosted.org/packages/94/c1/5160c9813269e39ae14b73debb907bfaaa1beee1762da8c4fb95df4764ed/multidict-6.6.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4bb7627fd7a968f41905a4d6343b0d63244a0623f006e9ed989fa2b78f4438a0", size = 251613, upload-time = "2025-08-11T12:06:13.97Z" },
-    { url = "https://files.pythonhosted.org/packages/05/a9/48d1bd111fc2f8fb98b2ed7f9a115c55a9355358432a19f53c0b74d8425d/multidict-6.6.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caebafea30ed049c57c673d0b36238b1748683be2593965614d7b0e99125c877", size = 241649, upload-time = "2025-08-11T12:06:15.204Z" },
-    { url = "https://files.pythonhosted.org/packages/85/2a/f7d743df0019408768af8a70d2037546a2be7b81fbb65f040d76caafd4c5/multidict-6.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ad887a8250eb47d3ab083d2f98db7f48098d13d42eb7a3b67d8a5c795f224ace", size = 239238, upload-time = "2025-08-11T12:06:16.467Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/b8/4f4bb13323c2d647323f7919201493cf48ebe7ded971717bfb0f1a79b6bf/multidict-6.6.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ed8358ae7d94ffb7c397cecb62cbac9578a83ecefc1eba27b9090ee910e2efb6", size = 233517, upload-time = "2025-08-11T12:06:18.107Z" },
-    { url = "https://files.pythonhosted.org/packages/33/29/4293c26029ebfbba4f574febd2ed01b6f619cfa0d2e344217d53eef34192/multidict-6.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecab51ad2462197a4c000b6d5701fc8585b80eecb90583635d7e327b7b6923eb", size = 243122, upload-time = "2025-08-11T12:06:19.361Z" },
-    { url = "https://files.pythonhosted.org/packages/20/60/a1c53628168aa22447bfde3a8730096ac28086704a0d8c590f3b63388d0c/multidict-6.6.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c5c97aa666cf70e667dfa5af945424ba1329af5dd988a437efeb3a09430389fb", size = 248992, upload-time = "2025-08-11T12:06:20.661Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/3b/55443a0c372f33cae5d9ec37a6a973802884fa0ab3586659b197cf8cc5e9/multidict-6.6.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9a950b7cf54099c1209f455ac5970b1ea81410f2af60ed9eb3c3f14f0bfcf987", size = 243708, upload-time = "2025-08-11T12:06:21.891Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/60/a18c6900086769312560b2626b18e8cca22d9e85b1186ba77f4755b11266/multidict-6.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:163c7ea522ea9365a8a57832dea7618e6cbdc3cd75f8c627663587459a4e328f", size = 237498, upload-time = "2025-08-11T12:06:23.206Z" },
-    { url = "https://files.pythonhosted.org/packages/11/3d/8bdd8bcaff2951ce2affccca107a404925a2beafedd5aef0b5e4a71120a6/multidict-6.6.4-cp310-cp310-win32.whl", hash = "sha256:17d2cbbfa6ff20821396b25890f155f40c986f9cfbce5667759696d83504954f", size = 41415, upload-time = "2025-08-11T12:06:24.77Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/53/cab1ad80356a4cd1b685a254b680167059b433b573e53872fab245e9fc95/multidict-6.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ce9a40fbe52e57e7edf20113a4eaddfacac0561a0879734e636aa6d4bb5e3fb0", size = 46046, upload-time = "2025-08-11T12:06:25.893Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/9a/874212b6f5c1c2d870d0a7adc5bb4cfe9b0624fa15cdf5cf757c0f5087ae/multidict-6.6.4-cp310-cp310-win_arm64.whl", hash = "sha256:01d0959807a451fe9fdd4da3e139cb5b77f7328baf2140feeaf233e1d777b729", size = 43147, upload-time = "2025-08-11T12:06:27.534Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" },
-    { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" },
-    { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" },
-    { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" },
-    { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" },
-    { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" },
-    { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" },
-    { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" },
-    { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" },
-    { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" },
-    { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" },
-    { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" },
-    { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" },
-    { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" },
-    { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" },
-    { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" },
-    { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" },
-    { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" },
-    { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" },
-    { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" },
-    { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848, upload-time = "2025-08-11T12:07:19.912Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060, upload-time = "2025-08-11T12:07:21.163Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269, upload-time = "2025-08-11T12:07:22.392Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158, upload-time = "2025-08-11T12:07:23.636Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076, upload-time = "2025-08-11T12:07:25.049Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694, upload-time = "2025-08-11T12:07:26.458Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350, upload-time = "2025-08-11T12:07:27.94Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250, upload-time = "2025-08-11T12:07:29.303Z" },
-    { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900, upload-time = "2025-08-11T12:07:30.764Z" },
-    { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355, upload-time = "2025-08-11T12:07:32.205Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061, upload-time = "2025-08-11T12:07:33.623Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675, upload-time = "2025-08-11T12:07:34.958Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247, upload-time = "2025-08-11T12:07:36.588Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960, upload-time = "2025-08-11T12:07:39.735Z" },
-    { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078, upload-time = "2025-08-11T12:07:41.525Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708, upload-time = "2025-08-11T12:07:43.405Z" },
-    { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912, upload-time = "2025-08-11T12:07:45.082Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076, upload-time = "2025-08-11T12:07:46.746Z" },
-    { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812, upload-time = "2025-08-11T12:07:48.402Z" },
-    { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313, upload-time = "2025-08-11T12:07:49.679Z" },
-    { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777, upload-time = "2025-08-11T12:07:51.318Z" },
-    { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321, upload-time = "2025-08-11T12:07:52.965Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954, upload-time = "2025-08-11T12:07:54.423Z" },
-    { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612, upload-time = "2025-08-11T12:07:55.914Z" },
-    { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528, upload-time = "2025-08-11T12:07:57.371Z" },
-    { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329, upload-time = "2025-08-11T12:07:58.844Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928, upload-time = "2025-08-11T12:08:01.037Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228, upload-time = "2025-08-11T12:08:02.96Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869, upload-time = "2025-08-11T12:08:04.746Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446, upload-time = "2025-08-11T12:08:06.332Z" },
-    { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299, upload-time = "2025-08-11T12:08:07.931Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926, upload-time = "2025-08-11T12:08:09.467Z" },
-    { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383, upload-time = "2025-08-11T12:08:10.981Z" },
-    { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" },
-    { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/84/0b/19348d4c98980c4851d2f943f8ebafdece2ae7ef737adcfa5994ce8e5f10/multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5", size = 77176, upload-time = "2026-01-26T02:42:59.784Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/04/9de3f8077852e3d438215c81e9b691244532d2e05b4270e89ce67b7d103c/multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8", size = 44996, upload-time = "2026-01-26T02:43:01.674Z" },
+    { url = "https://files.pythonhosted.org/packages/31/5c/08c7f7fe311f32e83f7621cd3f99d805f45519cd06fafb247628b861da7d/multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872", size = 44631, upload-time = "2026-01-26T02:43:03.169Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/7f/0e3b1390ae772f27501199996b94b52ceeb64fe6f9120a32c6c3f6b781be/multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991", size = 242561, upload-time = "2026-01-26T02:43:04.733Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/f4/8719f4f167586af317b69dd3e90f913416c91ca610cac79a45c53f590312/multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03", size = 242223, upload-time = "2026-01-26T02:43:06.695Z" },
+    { url = "https://files.pythonhosted.org/packages/47/ab/7c36164cce64a6ad19c6d9a85377b7178ecf3b89f8fd589c73381a5eedfd/multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981", size = 222322, upload-time = "2026-01-26T02:43:08.472Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/79/a25add6fb38035b5337bc5734f296d9afc99163403bbcf56d4170f97eb62/multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6", size = 254005, upload-time = "2026-01-26T02:43:10.127Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/7b/64a87cf98e12f756fc8bd444b001232ffff2be37288f018ad0d3f0aae931/multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190", size = 251173, upload-time = "2026-01-26T02:43:11.731Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92", size = 243273, upload-time = "2026-01-26T02:43:13.063Z" },
+    { url = "https://files.pythonhosted.org/packages/03/65/11492d6a0e259783720f3bc1d9ea55579a76f1407e31ed44045c99542004/multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee", size = 238956, upload-time = "2026-01-26T02:43:14.843Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/a7/7ee591302af64e7c196fb63fe856c788993c1372df765102bd0448e7e165/multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2", size = 233477, upload-time = "2026-01-26T02:43:16.025Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/99/c109962d58756c35fd9992fed7f2355303846ea2ff054bb5f5e9d6b888de/multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568", size = 243615, upload-time = "2026-01-26T02:43:17.84Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/5f/1973e7c771c86e93dcfe1c9cc55a5481b610f6614acfc28c0d326fe6bfad/multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40", size = 249930, upload-time = "2026-01-26T02:43:19.06Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/a5/f170fc2268c3243853580203378cd522446b2df632061e0a5409817854c7/multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962", size = 243807, upload-time = "2026-01-26T02:43:20.286Z" },
+    { url = "https://files.pythonhosted.org/packages/de/01/73856fab6d125e5bc652c3986b90e8699a95e84b48d72f39ade6c0e74a8c/multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505", size = 239103, upload-time = "2026-01-26T02:43:21.508Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/46/f1220bd9944d8aa40d8ccff100eeeee19b505b857b6f603d6078cb5315b0/multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122", size = 41416, upload-time = "2026-01-26T02:43:22.703Z" },
+    { url = "https://files.pythonhosted.org/packages/68/00/9b38e272a770303692fc406c36e1a4c740f401522d5787691eb38a8925a8/multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df", size = 46022, upload-time = "2026-01-26T02:43:23.77Z" },
+    { url = "https://files.pythonhosted.org/packages/64/65/d8d42490c02ee07b6bbe00f7190d70bb4738b3cce7629aaf9f213ef730dd/multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db", size = 43238, upload-time = "2026-01-26T02:43:24.882Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" },
+    { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" },
+    { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" },
+    { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" },
+    { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" },
+    { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" },
+    { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" },
+    { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" },
+    { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" },
+    { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" },
+    { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" },
+    { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" },
+    { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" },
+    { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" },
+    { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" },
+    { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" },
+    { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" },
+    { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" },
+    { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" },
+    { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" },
+    { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" },
+    { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" },
+    { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" },
+    { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" },
+    { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" },
+    { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" },
+    { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" },
+    { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" },
+    { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" },
+    { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" },
+    { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" },
+    { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" },
 ]
 
 [[package]]
 name = "multiprocess"
-version = "0.70.16"
+version = "0.70.18"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "dill" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980, upload-time = "2024-01-28T18:52:15.731Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982, upload-time = "2024-01-28T18:52:17.783Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" },
-    { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" },
-    { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" },
-]
-
-[[package]]
-name = "namex"
-version = "0.1.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/f8/7f9a8f08bf98cea1dfaa181e05cc8bbcb59cecf044b5a9ac3cce39f9c449/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:25d4012dcaaf66b9e8e955f58482b42910c2ee526d532844d8bcf661bbc604df", size = 135083, upload-time = "2025-04-17T03:11:04.223Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/03/b7b10dbfc17b2b3ce07d4d30b3ba8367d0ed32d6d46cd166e298f161dd46/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:06b19433de0d02afe5869aec8931dd5c01d99074664f806c73896b0d9e527213", size = 135128, upload-time = "2025-04-17T03:11:06.045Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/a3/5f8d3b9690ea5580bee5868ab7d7e2cfca74b7e826b28192b40aa3881cdc/multiprocess-0.70.18-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6fa1366f994373aaf2d4738b0f56e707caeaa05486e97a7f71ee0853823180c2", size = 135132, upload-time = "2025-04-17T03:11:07.533Z" },
+    { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" },
+    { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917, upload-time = "2025-04-17T03:11:24.044Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" },
 ]
 
 [[package]]
@@ -1348,7 +1407,7 @@ wheels = [
 
 [[package]]
 name = "networkx"
-version = "3.5"
+version = "3.6.1"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
     "python_full_version >= '3.14'",
@@ -1356,18 +1415,18 @@ resolution-markers = [
     "python_full_version == '3.12.*'",
     "python_full_version == '3.11.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" },
 ]
 
 [[package]]
 name = "nodeenv"
-version = "1.9.1"
+version = "1.10.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" },
+    { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" },
 ]
 
 [[package]]
@@ -1437,89 +1496,140 @@ wheels = [
 
 [[package]]
 name = "numpy"
-version = "2.3.3"
+version = "2.4.6"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version == '3.11.*'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" },
+    { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" },
+    { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" },
+    { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" },
+    { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" },
+    { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" },
+    { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" },
+    { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" },
+    { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" },
+    { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" },
+    { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" },
+    { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" },
+    { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" },
+    { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" },
+    { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" },
+    { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" },
+    { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" },
+    { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" },
+    { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" },
+    { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" },
+    { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" },
+    { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" },
+    { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" },
+    { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" },
+    { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" },
+    { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" },
+    { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" },
+    { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" },
+    { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" },
+    { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" },
+    { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" },
+]
+
+[[package]]
+name = "numpy"
+version = "2.5.1"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
     "python_full_version >= '3.14'",
     "python_full_version == '3.13.*'",
     "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253, upload-time = "2025-09-09T15:56:02.094Z" },
-    { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980, upload-time = "2025-09-09T15:56:05.926Z" },
-    { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709, upload-time = "2025-09-09T15:56:07.95Z" },
-    { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923, upload-time = "2025-09-09T15:56:09.443Z" },
-    { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591, upload-time = "2025-09-09T15:56:11.234Z" },
-    { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714, upload-time = "2025-09-09T15:56:14.637Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592, upload-time = "2025-09-09T15:56:17.285Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474, upload-time = "2025-09-09T15:56:20.943Z" },
-    { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794, upload-time = "2025-09-09T15:56:23.258Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104, upload-time = "2025-09-09T15:56:25.476Z" },
-    { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772, upload-time = "2025-09-09T15:56:27.679Z" },
-    { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014, upload-time = "2025-09-09T15:56:29.966Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220, upload-time = "2025-09-09T15:56:32.175Z" },
-    { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918, upload-time = "2025-09-09T15:56:34.175Z" },
-    { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922, upload-time = "2025-09-09T15:56:36.149Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991, upload-time = "2025-09-09T15:56:40.548Z" },
-    { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643, upload-time = "2025-09-09T15:56:43.343Z" },
-    { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787, upload-time = "2025-09-09T15:56:46.141Z" },
-    { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598, upload-time = "2025-09-09T15:56:49.844Z" },
-    { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800, upload-time = "2025-09-09T15:56:52.499Z" },
-    { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615, upload-time = "2025-09-09T15:56:54.422Z" },
-    { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936, upload-time = "2025-09-09T15:56:56.541Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588, upload-time = "2025-09-09T15:56:59.087Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802, upload-time = "2025-09-09T15:57:01.73Z" },
-    { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537, upload-time = "2025-09-09T15:57:03.765Z" },
-    { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743, upload-time = "2025-09-09T15:57:07.921Z" },
-    { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881, upload-time = "2025-09-09T15:57:11.349Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301, upload-time = "2025-09-09T15:57:14.245Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645, upload-time = "2025-09-09T15:57:16.534Z" },
-    { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179, upload-time = "2025-09-09T15:57:18.883Z" },
-    { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250, upload-time = "2025-09-09T15:57:21.296Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269, upload-time = "2025-09-09T15:57:23.034Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314, upload-time = "2025-09-09T15:57:25.045Z" },
-    { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025, upload-time = "2025-09-09T15:57:27.257Z" },
-    { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053, upload-time = "2025-09-09T15:57:30.077Z" },
-    { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444, upload-time = "2025-09-09T15:57:32.733Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039, upload-time = "2025-09-09T15:57:34.328Z" },
-    { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314, upload-time = "2025-09-09T15:57:36.255Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722, upload-time = "2025-09-09T15:57:38.622Z" },
-    { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755, upload-time = "2025-09-09T15:57:41.16Z" },
-    { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560, upload-time = "2025-09-09T15:57:43.459Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776, upload-time = "2025-09-09T15:57:45.793Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281, upload-time = "2025-09-09T15:57:47.492Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275, upload-time = "2025-09-09T15:57:49.647Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527, upload-time = "2025-09-09T15:57:52.006Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159, upload-time = "2025-09-09T15:57:54.407Z" },
-    { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624, upload-time = "2025-09-09T15:57:56.5Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627, upload-time = "2025-09-09T15:57:58.206Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926, upload-time = "2025-09-09T15:58:00.035Z" },
-    { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958, upload-time = "2025-09-09T15:58:02.738Z" },
-    { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920, upload-time = "2025-09-09T15:58:05.029Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076, upload-time = "2025-09-09T15:58:07.745Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952, upload-time = "2025-09-09T15:58:10.096Z" },
-    { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322, upload-time = "2025-09-09T15:58:12.138Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630, upload-time = "2025-09-09T15:58:14.64Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987, upload-time = "2025-09-09T15:58:16.889Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076, upload-time = "2025-09-09T15:58:20.343Z" },
-    { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491, upload-time = "2025-09-09T15:58:22.481Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913, upload-time = "2025-09-09T15:58:24.569Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811, upload-time = "2025-09-09T15:58:26.416Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689, upload-time = "2025-09-09T15:58:28.831Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855, upload-time = "2025-09-09T15:58:31.349Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520, upload-time = "2025-09-09T15:58:33.762Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371, upload-time = "2025-09-09T15:58:36.04Z" },
-    { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576, upload-time = "2025-09-09T15:58:37.927Z" },
-    { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953, upload-time = "2025-09-09T15:58:40.576Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019, upload-time = "2025-09-09T15:58:42.838Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288, upload-time = "2025-09-09T15:58:45.425Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425, upload-time = "2025-09-09T15:58:48.6Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053, upload-time = "2025-09-09T15:58:50.401Z" },
-    { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354, upload-time = "2025-09-09T15:58:52.704Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413, upload-time = "2025-09-09T15:58:55.027Z" },
-    { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844, upload-time = "2025-09-09T15:58:57.359Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" },
+    { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" },
+    { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" },
+    { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" },
+    { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" },
+    { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" },
+    { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" },
+    { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" },
+    { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" },
+    { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" },
+    { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" },
+    { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" },
+    { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" },
+    { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" },
+    { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" },
+    { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" },
+    { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" },
+    { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" },
+    { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" },
+    { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" },
+    { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" },
 ]
 
 [[package]]
@@ -1649,11 +1759,11 @@ wheels = [
 
 [[package]]
 name = "nvidia-nvjitlink"
-version = "13.0.88"
+version = "13.3.33"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" },
-    { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" },
+    { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" },
 ]
 
 [[package]]
@@ -1675,73 +1785,52 @@ wheels = [
 ]
 
 [[package]]
-name = "opt-einsum"
-version = "3.4.0"
+name = "opentelemetry-api"
+version = "1.30.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" }
+dependencies = [
+    { name = "deprecated" },
+    { name = "importlib-metadata" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2b/6d/bbbf879826b7f3c89a45252010b5796fb1f1a0d45d9dc4709db0ef9a06c8/opentelemetry_api-1.30.0.tar.gz", hash = "sha256:375893400c1435bf623f7dfb3bcd44825fe6b56c34d0667c542ea8257b1a1240", size = 63703, upload-time = "2025-02-04T18:17:13.789Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" },
+    { url = "https://files.pythonhosted.org/packages/36/0a/eea862fae6413d8181b23acf8e13489c90a45f17986ee9cf4eab8a0b9ad9/opentelemetry_api-1.30.0-py3-none-any.whl", hash = "sha256:d5f5284890d73fdf47f843dda3210edf37a38d66f44f2b5aedc1e89ed455dc09", size = 64955, upload-time = "2025-02-04T18:16:46.167Z" },
 ]
 
 [[package]]
-name = "optree"
-version = "0.17.0"
+name = "opentelemetry-sdk"
+version = "1.30.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-semantic-conventions" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111, upload-time = "2025-07-25T11:26:11.586Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398, upload-time = "2025-07-25T11:24:26.672Z" },
-    { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834, upload-time = "2025-07-25T11:24:28.988Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116, upload-time = "2025-07-25T11:24:30.396Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491, upload-time = "2025-07-25T11:24:31.725Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957, upload-time = "2025-07-25T11:24:32.759Z" },
-    { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797, upload-time = "2025-07-25T11:24:39.509Z" },
-    { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450, upload-time = "2025-07-25T11:24:40.555Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557, upload-time = "2025-07-25T11:24:42.396Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174, upload-time = "2025-07-25T11:24:43.51Z" },
-    { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000, upload-time = "2025-07-25T11:24:44.819Z" },
-    { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869, upload-time = "2025-07-25T11:24:51.807Z" },
-    { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465, upload-time = "2025-07-25T11:24:53.124Z" },
-    { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686, upload-time = "2025-07-25T11:24:54.212Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225, upload-time = "2025-07-25T11:24:55.213Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819, upload-time = "2025-07-25T11:24:56.315Z" },
-    { url = "https://files.pythonhosted.org/packages/21/04/9706d11b880186e9e9d66d7c21ce249b2ce0212645137cc13fdd18247c26/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5995a3efce4b00a14049268a81ab0379656a41ddf3c3761e3b88937fca44d48", size = 348177, upload-time = "2025-07-25T11:25:00.999Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/4b/0415c18816818ac871c9f3d5c7c5f4ceb83baff03ed511c9c94591ace4bc/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d06e8143d16fe6c0708f3cc2807b5b65f815d60ee2b52f3d79e4022c95563482", size = 354389, upload-time = "2025-07-25T11:25:02.337Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513, upload-time = "2025-07-25T11:25:05.548Z" },
-    { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378, upload-time = "2025-07-25T11:25:06.918Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294, upload-time = "2025-07-25T11:25:08.043Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633, upload-time = "2025-07-25T11:25:09.141Z" },
-    { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866, upload-time = "2025-07-25T11:25:10.214Z" },
-    { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945, upload-time = "2025-07-25T11:25:17.728Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177, upload-time = "2025-07-25T11:25:18.749Z" },
-    { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198, upload-time = "2025-07-25T11:25:19.865Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391, upload-time = "2025-07-25T11:25:20.942Z" },
-    { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555, upload-time = "2025-07-25T11:25:21.968Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929, upload-time = "2025-07-25T11:25:28.897Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381, upload-time = "2025-07-25T11:25:29.984Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461, upload-time = "2025-07-25T11:25:31.116Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234, upload-time = "2025-07-25T11:25:32.289Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863, upload-time = "2025-07-25T11:25:33.379Z" },
-    { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975, upload-time = "2025-07-25T11:25:40.376Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173, upload-time = "2025-07-25T11:25:41.474Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080, upload-time = "2025-07-25T11:25:42.605Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422, upload-time = "2025-07-25T11:25:43.672Z" },
-    { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579, upload-time = "2025-07-25T11:25:44.765Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952, upload-time = "2025-07-25T11:26:02.692Z" },
-    { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568, upload-time = "2025-07-25T11:26:04.094Z" },
-    { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728, upload-time = "2025-07-25T11:26:07.959Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649, upload-time = "2025-07-25T11:26:09.463Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/93/ee/d710062e8a862433d1be0b85920d0c653abe318878fef2d14dfe2c62ff7b/opentelemetry_sdk-1.30.0.tar.gz", hash = "sha256:c9287a9e4a7614b9946e933a67168450b9ab35f08797eb9bc77d998fa480fa18", size = 158633, upload-time = "2025-02-04T18:17:28.908Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/97/28/64d781d6adc6bda2260067ce2902bd030cf45aec657e02e28c5b4480b976/opentelemetry_sdk-1.30.0-py3-none-any.whl", hash = "sha256:14fe7afc090caad881addb6926cec967129bd9260c4d33ae6a217359f6b61091", size = 118717, upload-time = "2025-02-04T18:17:09.353Z" },
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.51b0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "deprecated" },
+    { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1e/c0/0f9ef4605fea7f2b83d55dd0b0d7aebe8feead247cd6facd232b30907b4f/opentelemetry_semantic_conventions-0.51b0.tar.gz", hash = "sha256:3fabf47f35d1fd9aebcdca7e6802d86bd5ebc3bc3408b7e3248dde6e87a18c47", size = 107191, upload-time = "2025-02-04T18:17:29.903Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/75/d7bdbb6fd8630b4cafb883482b75c4fc276b6426619539d266e32ac53266/opentelemetry_semantic_conventions-0.51b0-py3-none-any.whl", hash = "sha256:fdc777359418e8d06c86012c3dc92c88a6453ba662e941593adb062e48c2eeae", size = 177416, upload-time = "2025-02-04T18:17:11.305Z" },
 ]
 
 [[package]]
 name = "packaging"
-version = "25.0"
+version = "26.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
+    { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" },
 ]
 
 [[package]]
@@ -1750,7 +1839,8 @@ version = "2.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
+    { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "python-dateutil" },
     { name = "pytz" },
     { name = "tzdata" },
@@ -1954,102 +2044,130 @@ wheels = [
 
 [[package]]
 name = "propcache"
-version = "0.3.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178, upload-time = "2025-06-09T22:53:40.126Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133, upload-time = "2025-06-09T22:53:41.965Z" },
-    { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039, upload-time = "2025-06-09T22:53:43.268Z" },
-    { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903, upload-time = "2025-06-09T22:53:44.872Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362, upload-time = "2025-06-09T22:53:46.707Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525, upload-time = "2025-06-09T22:53:48.547Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283, upload-time = "2025-06-09T22:53:50.067Z" },
-    { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872, upload-time = "2025-06-09T22:53:51.438Z" },
-    { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452, upload-time = "2025-06-09T22:53:53.229Z" },
-    { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567, upload-time = "2025-06-09T22:53:54.541Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015, upload-time = "2025-06-09T22:53:56.44Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660, upload-time = "2025-06-09T22:53:57.839Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105, upload-time = "2025-06-09T22:53:59.638Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980, upload-time = "2025-06-09T22:54:01.071Z" },
-    { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679, upload-time = "2025-06-09T22:54:03.003Z" },
-    { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459, upload-time = "2025-06-09T22:54:04.134Z" },
-    { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207, upload-time = "2025-06-09T22:54:05.399Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648, upload-time = "2025-06-09T22:54:08.023Z" },
-    { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496, upload-time = "2025-06-09T22:54:09.228Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288, upload-time = "2025-06-09T22:54:10.466Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456, upload-time = "2025-06-09T22:54:11.828Z" },
-    { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429, upload-time = "2025-06-09T22:54:13.823Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472, upload-time = "2025-06-09T22:54:15.232Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480, upload-time = "2025-06-09T22:54:17.104Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530, upload-time = "2025-06-09T22:54:18.512Z" },
-    { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230, upload-time = "2025-06-09T22:54:19.947Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754, upload-time = "2025-06-09T22:54:21.716Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430, upload-time = "2025-06-09T22:54:23.17Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884, upload-time = "2025-06-09T22:54:25.539Z" },
-    { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480, upload-time = "2025-06-09T22:54:26.892Z" },
-    { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757, upload-time = "2025-06-09T22:54:28.241Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500, upload-time = "2025-06-09T22:54:29.4Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" },
-    { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" },
-    { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" },
-    { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" },
-    { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" },
-    { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" },
-    { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" },
-    { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" },
-    { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" },
-    { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" },
-    { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" },
-    { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" },
-    { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" },
-    { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" },
-    { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" },
-    { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" },
-]
-
-[[package]]
-name = "protobuf"
-version = "6.32.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" },
-    { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" },
+version = "0.5.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5b/56/030b7b4719d53085722893e0009dffb9236aa10bca1b12121bdc5626ef16/propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b", size = 93417, upload-time = "2026-05-08T20:59:15.597Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/55/1140a8e067b8ec093a18a4ae7bb0045d9db65da38a08618ddc5e2f1994aa/propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c", size = 53847, upload-time = "2026-05-08T20:59:17.096Z" },
+    { url = "https://files.pythonhosted.org/packages/20/42/0e7443c90310498561addf346e7d57fe3c6ba1914e1ba938b5464c7bbfd2/propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb", size = 53512, upload-time = "2026-05-08T20:59:18.64Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/db/cf51a71bab2009517d1a7f0ee07657e3bd446c4d69f67e6966cf17bcf956/propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e", size = 58068, upload-time = "2026-05-08T20:59:20.683Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/43/39b6bdee9699fa1e1641c519feeb64a67e2a9f93bb465c70776b37a7333f/propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e", size = 61020, upload-time = "2026-05-08T20:59:22.112Z" },
+    { url = "https://files.pythonhosted.org/packages/26/0b/843726fbb0a29a8c5684fdb25971823638399f31e52e9d1f06a02dc9aa6b/propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b", size = 62732, upload-time = "2026-05-08T20:59:23.805Z" },
+    { url = "https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d", size = 60140, upload-time = "2026-05-08T20:59:25.389Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/09/3da4be9b5b879219ad234aa535b3dd4a080ed1ad48d3a73ca07a9e798f22/propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d", size = 60400, upload-time = "2026-05-08T20:59:27.238Z" },
+    { url = "https://files.pythonhosted.org/packages/60/2f/09b72b874a9aa0044faf52a69807a6ed618e267ceaa9ec4a63195fa5b504/propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0", size = 58155, upload-time = "2026-05-08T20:59:28.48Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/37/97489848c54c95578045473954f10956d619ce6a09e7ac137b71cdcb698b/propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b", size = 57037, upload-time = "2026-05-08T20:59:30.146Z" },
+    { url = "https://files.pythonhosted.org/packages/22/db/6c695285ccfc49012743ee9c98212b8c5dd0aed7b63cfd816d4a0f7a1601/propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf", size = 61103, upload-time = "2026-05-08T20:59:31.626Z" },
+    { url = "https://files.pythonhosted.org/packages/98/a9/1e500401ca593b0bdb6bf75a70bc2d723835fd53360edff6af70692c7546/propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf", size = 60394, upload-time = "2026-05-08T20:59:32.829Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/87/f638b6e375eae0f30a1a2325d8b34fd85fdc785bb9960cf805f3bf1ec69a/propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e", size = 63084, upload-time = "2026-05-08T20:59:35.964Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/18/884573f5d97b6d9eba68de759a82c901b7e39d7904d30f7b8d58d42d2a12/propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274", size = 60999, upload-time = "2026-05-08T20:59:38.481Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/1a/c3915eb059ceec9e758a56e4cfd955292bc0f201be2176a46b76d94b303a/propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe", size = 39036, upload-time = "2026-05-08T20:59:40.323Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/02/1dfd5607501a602d19c1c449d2d193b7d1c611f9246b4059026a1189a80e/propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d", size = 42190, upload-time = "2026-05-08T20:59:42.232Z" },
+    { url = "https://files.pythonhosted.org/packages/57/93/f71588ad08b3e6f4b555b5ef215808a3c02b042d0151ad82fa6f15be677a/propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5", size = 38545, upload-time = "2026-05-08T20:59:44.087Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" },
+    { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" },
+    { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" },
+    { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" },
+    { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" },
+    { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" },
+    { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" },
+    { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" },
+    { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" },
+    { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" },
+    { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" },
+    { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" },
+    { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" },
+    { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" },
+    { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" },
+    { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" },
+    { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" },
+    { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" },
+    { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" },
+    { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" },
+    { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" },
+    { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" },
+    { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" },
+    { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" },
+    { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" },
+    { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" },
+    { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" },
+    { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" },
+    { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" },
+    { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" },
+    { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" },
+    { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" },
+    { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" },
+    { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" },
+    { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" },
+    { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" },
+    { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" },
+    { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" },
+    { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" },
 ]
 
 [[package]]
@@ -2079,64 +2197,57 @@ wheels = [
 
 [[package]]
 name = "pyarrow"
-version = "24.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" },
-    { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" },
-    { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" },
-    { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" },
-    { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" },
-    { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" },
-    { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" },
-    { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" },
-    { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" },
-    { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" },
-    { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" },
-    { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" },
-    { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" },
-    { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" },
-    { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" },
-    { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" },
-    { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" },
-    { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" },
-    { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" },
-    { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" },
-    { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" },
-    { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" },
-    { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" },
+version = "25.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4d/2a/eaa70e6d6ed430c2e90c0599e2831a41a50251879e44788ccdbc73115af1/pyarrow-25.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ce0ca222802087b9a8cb031a6468442cb6b67c290a45a601cac64753d34954d3", size = 35945551, upload-time = "2026-07-10T08:25:23.153Z" },
+    { url = "https://files.pythonhosted.org/packages/df/e0/917086af6b246143012cdc8a7c886b018b53204f3d69fc5f9be5857a8b80/pyarrow-25.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7d6da02ffc7a3a9bda3b7ded4cc2a27ff73969ab37153f3afd46bbbc1ba4f0f7", size = 37636698, upload-time = "2026-07-10T08:25:28.031Z" },
+    { url = "https://files.pythonhosted.org/packages/68/6a/c87829f92503f84993721791c942f3d9aa81044de51a8cfb1da5810e5345/pyarrow-25.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:dbf9fa5d4bde73b1cc16377dcaaa010f971e6fa7f5083f5d44f34b50bc1d74af", size = 46858364, upload-time = "2026-07-10T08:25:34.527Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/ba/2030d454c2747e26cce23e4a0338067ee0830a155b7894da04caa96783a5/pyarrow-25.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b72d943ff4e10fec8d48aedb23322d8f6ea8bc2d698b81db37e73730f69e4862", size = 50056398, upload-time = "2026-07-10T08:25:40.785Z" },
+    { url = "https://files.pythonhosted.org/packages/78/ce/ba7a5ce7bf0cfc372ec48203a34ece42f73aa2f3231706f61c55e105ecd0/pyarrow-25.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5fb2d837960f1df7f679ff9f1a55065e306347d379e0768cebf14781254d6194", size = 49958146, upload-time = "2026-07-10T08:25:46.98Z" },
+    { url = "https://files.pythonhosted.org/packages/75/eb/c34a29fb7a70dca2f903c7d85a928928ef55af20cd56e99de6b4c0d897bc/pyarrow-25.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:add690feafa0953c443cdba9e9e87f5eaa198f1ea2e43a3b146ea83f202262d0", size = 53096264, upload-time = "2026-07-10T08:25:53.925Z" },
+    { url = "https://files.pythonhosted.org/packages/36/f9/35b1f83a0727d84951588e4034aca2feb76dfb45b0725918c0037b0a48f7/pyarrow-25.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d293e9959b29a24c82d936d04ab2b7fd8b8d334030de2e56a99aba94f008ad7a", size = 27840572, upload-time = "2026-07-10T08:25:58.966Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" },
+    { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" },
+    { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" },
+    { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" },
+    { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" },
+    { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" },
+    { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" },
+    { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" },
+    { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" },
+    { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" },
+    { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" },
+    { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" },
+    { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" },
+    { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" },
+    { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" },
+    { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" },
+    { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" },
+    { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" },
 ]
 
 [[package]]
 name = "pydantic"
-version = "2.12.4"
+version = "2.13.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "annotated-types" },
@@ -2144,136 +2255,134 @@ dependencies = [
     { name = "typing-extensions" },
     { name = "typing-inspection" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" },
 ]
 
 [[package]]
 name = "pydantic-core"
-version = "2.41.5"
+version = "2.46.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" },
-    { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" },
-    { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" },
-    { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" },
-    { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" },
-    { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" },
-    { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" },
-    { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" },
-    { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" },
-    { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" },
-    { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" },
-    { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" },
-    { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" },
-    { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" },
-    { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" },
-    { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" },
-    { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" },
-    { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" },
-    { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" },
-    { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" },
-    { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" },
-    { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" },
-    { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" },
-    { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" },
-    { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" },
-    { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" },
-    { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" },
-    { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" },
-    { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" },
-    { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" },
-    { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" },
-    { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" },
-    { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" },
-    { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" },
-    { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" },
-    { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" },
-    { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" },
-    { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" },
-    { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" },
-    { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" },
-    { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" },
-    { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" },
-    { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" },
-    { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" },
-    { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" },
+    { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" },
+    { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" },
+    { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" },
+    { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" },
+    { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" },
+    { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" },
+    { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" },
+    { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" },
+    { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" },
+    { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" },
+    { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" },
+    { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" },
+    { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" },
+    { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" },
+    { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" },
+    { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" },
+    { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" },
+    { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" },
+    { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" },
+    { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" },
+    { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" },
+    { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" },
+    { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" },
+    { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" },
+    { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" },
+    { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" },
+    { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" },
+    { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" },
+    { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" },
+    { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" },
+    { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" },
+    { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" },
+    { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" },
+    { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" },
+    { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" },
+    { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" },
+    { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" },
+    { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" },
+    { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" },
+    { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" },
+    { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" },
+    { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" },
+    { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" },
+    { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" },
 ]
 
 [[package]]
 name = "pygments"
-version = "2.19.2"
+version = "2.20.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
 ]
 
 [[package]]
@@ -2282,7 +2391,8 @@ source = { editable = "." }
 dependencies = [
     { name = "lance-namespace" },
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
+    { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "pyarrow" },
 ]
 
@@ -2298,6 +2408,10 @@ geo = [
     { name = "geoarrow-rust-core" },
     { name = "geoarrow-rust-io" },
 ]
+otel = [
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-sdk" },
+]
 tests = [
     { name = "boto3" },
     { name = "datafusion" },
@@ -2309,7 +2423,6 @@ tests = [
     { name = "polars", extra = ["pandas", "pyarrow"] },
     { name = "psutil" },
     { name = "pytest" },
-    { name = "tensorflow", marker = "sys_platform == 'linux'" },
     { name = "tqdm" },
 ]
 torch = [
@@ -2331,26 +2444,28 @@ tests = [
     { name = "datasets" },
     { name = "duckdb" },
     { name = "ml-dtypes" },
+    { name = "opentelemetry-sdk" },
     { name = "pandas" },
     { name = "pillow" },
     { name = "polars", extra = ["pandas", "pyarrow"] },
     { name = "psutil" },
     { name = "pytest" },
-    { name = "tensorflow", marker = "sys_platform == 'linux'" },
     { name = "tqdm" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "boto3", marker = "extra == 'tests'" },
-    { name = "datafusion", marker = "extra == 'tests'", specifier = ">=53,<54" },
-    { name = "datasets", marker = "extra == 'tests'" },
-    { name = "duckdb", marker = "extra == 'tests'" },
+    { name = "datafusion", marker = "extra == 'tests'", specifier = ">=54,<55" },
+    { name = "datasets", marker = "extra == 'tests'", specifier = "==4.4.0" },
+    { name = "duckdb", marker = "extra == 'tests'", specifier = ">=1.5.0,<1.6.0" },
     { name = "geoarrow-rust-core", marker = "extra == 'geo'" },
     { name = "geoarrow-rust-io", marker = "extra == 'geo'" },
     { name = "lance-namespace", specifier = ">=0.8.5,<0.9" },
     { name = "ml-dtypes", marker = "extra == 'tests'" },
     { name = "numpy", specifier = ">=1.22" },
+    { name = "opentelemetry-api", marker = "extra == 'otel'" },
+    { name = "opentelemetry-sdk", marker = "extra == 'otel'" },
     { name = "pandas", marker = "extra == 'tests'" },
     { name = "pillow", marker = "extra == 'tests'" },
     { name = "polars", extras = ["pyarrow", "pandas"], marker = "extra == 'tests'" },
@@ -2360,11 +2475,10 @@ requires-dist = [
     { name = "pytest", marker = "extra == 'tests'" },
     { name = "pytest-benchmark", marker = "extra == 'benchmarks'" },
     { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.2" },
-    { name = "tensorflow", marker = "sys_platform == 'linux' and extra == 'tests'" },
     { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0" },
     { name = "tqdm", marker = "extra == 'tests'" },
 ]
-provides-extras = ["benchmarks", "dev", "geo", "tests", "torch"]
+provides-extras = ["benchmarks", "dev", "geo", "otel", "tests", "torch"]
 
 [package.metadata.requires-dev]
 benchmarks = [{ name = "pytest-benchmark", specifier = "==5.1.0" }]
@@ -2375,16 +2489,16 @@ dev = [
 ]
 tests = [
     { name = "boto3", specifier = "==1.40.43" },
-    { name = "datafusion", specifier = "==53.0.0" },
-    { name = "datasets", specifier = "==4.1.1" },
-    { name = "duckdb", specifier = "==1.4.0" },
+    { name = "datafusion", specifier = "==54.0.0" },
+    { name = "datasets", specifier = "==4.4.0" },
+    { name = "duckdb", specifier = ">=1.5.0,<1.6.0" },
     { name = "ml-dtypes", specifier = "==0.5.3" },
+    { name = "opentelemetry-sdk", specifier = "==1.30.0" },
     { name = "pandas", specifier = "==2.3.3" },
     { name = "pillow", specifier = "==11.3.0" },
     { name = "polars", extras = ["pyarrow", "pandas"], specifier = "==1.34.0" },
     { name = "psutil", specifier = "==7.1.0" },
     { name = "pytest", specifier = "==8.4.2" },
-    { name = "tensorflow", marker = "sys_platform == 'linux'", specifier = "==2.20.0" },
     { name = "tqdm", specifier = "==4.67.1" },
 ]
 
@@ -2563,11 +2677,11 @@ wheels = [
 
 [[package]]
 name = "pytz"
-version = "2025.2"
+version = "2026.2"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" },
 ]
 
 [[package]]
@@ -2636,7 +2750,7 @@ wheels = [
 
 [[package]]
 name = "requests"
-version = "2.33.0"
+version = "2.34.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "certifi" },
@@ -2644,22 +2758,9 @@ dependencies = [
     { name = "idna" },
     { name = "urllib3" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" },
-]
-
-[[package]]
-name = "rich"
-version = "14.1.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "markdown-it-py" },
-    { name = "pygments" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" },
 ]
 
 [[package]]
@@ -2701,11 +2802,11 @@ wheels = [
 
 [[package]]
 name = "setuptools"
-version = "80.9.0"
+version = "83.0.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" },
 ]
 
 [[package]]
@@ -2729,170 +2830,106 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
 ]
 
-[[package]]
-name = "tensorboard"
-version = "2.20.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "absl-py" },
-    { name = "grpcio" },
-    { name = "markdown" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "packaging" },
-    { name = "pillow" },
-    { name = "protobuf" },
-    { name = "setuptools" },
-    { name = "tensorboard-data-server" },
-    { name = "werkzeug" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" },
-]
-
-[[package]]
-name = "tensorboard-data-server"
-version = "0.7.2"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" },
-    { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" },
-]
-
-[[package]]
-name = "tensorflow"
-version = "2.20.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "absl-py" },
-    { name = "astunparse" },
-    { name = "flatbuffers" },
-    { name = "gast" },
-    { name = "google-pasta" },
-    { name = "grpcio" },
-    { name = "h5py" },
-    { name = "keras" },
-    { name = "libclang" },
-    { name = "ml-dtypes" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "opt-einsum" },
-    { name = "packaging" },
-    { name = "protobuf" },
-    { name = "requests" },
-    { name = "setuptools" },
-    { name = "six" },
-    { name = "tensorboard" },
-    { name = "termcolor" },
-    { name = "typing-extensions" },
-    { name = "wrapt" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122, upload-time = "2025-08-13T16:50:47.909Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510, upload-time = "2025-08-13T16:51:02.608Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882, upload-time = "2025-08-13T16:51:39.546Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129, upload-time = "2025-08-13T16:51:55.104Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109, upload-time = "2025-08-13T16:52:31.513Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547, upload-time = "2025-08-13T16:52:46.396Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" },
-    { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" },
-]
-
-[[package]]
-name = "termcolor"
-version = "3.1.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" },
-]
-
 [[package]]
 name = "tomli"
-version = "2.2.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" },
-    { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" },
-    { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" },
-    { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" },
-    { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" },
-    { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" },
-    { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" },
-    { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" },
-    { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" },
-    { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" },
-    { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" },
-    { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" },
-    { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" },
+version = "2.4.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" },
+    { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" },
+    { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" },
+    { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" },
+    { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" },
+    { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" },
+    { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" },
+    { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" },
+    { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" },
+    { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" },
+    { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" },
+    { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" },
+    { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" },
+    { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" },
+    { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" },
+    { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" },
+    { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" },
+    { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" },
+    { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" },
+    { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" },
+    { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" },
+    { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" },
+    { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" },
+    { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" },
+    { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" },
+    { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" },
 ]
 
 [[package]]
 name = "torch"
-version = "2.12.1"
+version = "2.13.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cuda-bindings", marker = "sys_platform == 'linux'" },
-    { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" },
+    { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" },
+    { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" },
     { name = "filelock" },
     { name = "fsspec" },
     { name = "jinja2" },
     { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "nvidia-cublas", marker = "sys_platform == 'linux'" },
+    { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" },
     { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" },
     { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" },
     { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" },
     { name = "setuptools" },
     { name = "sympy" },
-    { name = "triton", marker = "sys_platform == 'linux'" },
+    { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" },
     { name = "typing-extensions" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/db/ed/ff0c4f8cef63977a646dc80e40c05cae873f4097b12dc87e1cd7e1cecf42/torch-2.12.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ec56e82be6a8b0c036771a77f7d32ad3c299770571af9815b3dafe61434389d5", size = 87967927, upload-time = "2026-06-17T21:08:43.16Z" },
-    { url = "https://files.pythonhosted.org/packages/85/1b/c8ecf60c9dba535f9ea341c359c600c0bd877a7ca14b3296f13316321847/torch-2.12.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:42cd7339bf266f14944710e8274be63e7e012bb937834a8d85a8327a9860eba6", size = 426366829, upload-time = "2026-06-17T21:07:18.574Z" },
-    { url = "https://files.pythonhosted.org/packages/ab/d6/73d4a3f27e00526e98086f3a64ab609af1345cca62367749fbc3c8e4b83c/torch-2.12.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a7817f0f89a796d9de239d06f69faf5d7e19a6a5db6710a5ead777c912f9f50a", size = 532144834, upload-time = "2026-06-17T21:08:00.633Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/51/4010c8fa6f9d1f42c054a321970ca95ec58e4e4494f5b53a34c3f3c9e310/torch-2.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:2af3d9cc866e0a15ae7635ff0a9c61d6624a353ad657f5bcd8d86c26cdc64693", size = 122949863, upload-time = "2026-06-17T21:08:39.016Z" },
-    { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/e3/750b3e3548635ceac03ba255daa26dbc7ed66ca3484dc4b4d955ab7f4501/torch-2.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:107df6888624bdea41508f9aeb6149d9333c737a5530ceecb56c904e811369ae", size = 426379894, upload-time = "2026-06-17T21:06:55.077Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/ca/ed24783da629ff3e640ba3f70a7639e9045d3d88b93ee6bc47b8a28a1f2c/torch-2.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:6e29e7e74d05bda7d955c75e99459f878ebd970ef851b4057edbd3b34a5eb4a3", size = 532169264, upload-time = "2026-06-17T21:08:17.65Z" },
-    { url = "https://files.pythonhosted.org/packages/46/61/c63f0158446f3a98ea672b004d761b848911eba567ea4a624c7db5aadc04/torch-2.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:a513506cfda3c1c78dabeb6574c1597538c0254b3d39af174dde35d8177f4ce3", size = 122953086, upload-time = "2026-06-17T21:08:27.69Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/54/efb7ebca77970012b0cc21687a55d70eb2ba514b2c2b8e18d9fb1222f3be/torch-2.12.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d2dd0f2c5f7ccbddaf34cade0deaf476808368f902b9cdb7f36a2ab42301bc0e", size = 87991951, upload-time = "2026-06-17T21:07:49.309Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/00/4210d76ca7424981f04033ebe7e48816ab83287a62538747a58825db770c/torch-2.12.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2de4e19b88a481482c6c75291f2d6a52eda3ce51f311b29aa9b68499c830c07c", size = 426382721, upload-time = "2026-06-17T21:06:41.842Z" },
-    { url = "https://files.pythonhosted.org/packages/76/1f/bc9f5a5aa569307076365f25afcebacb22e9c754b1bcfbaaa146627c7fda/torch-2.12.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:649e4ced014ba646f76f8cb9c9726735a6323eb321b7919f942790a923f90921", size = 532261322, upload-time = "2026-06-17T21:06:06.673Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/49/c549461daa008159d006a76a991fbc2f26fa8bac27a4030c858463dcb20f/torch-2.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:e86550597877fb272ddc52db2f85b82cb601ea7bd932576a0340152cae2200b3", size = 122988095, upload-time = "2026-06-17T21:07:44.9Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/4a/0300261818e1560d72cc160ac826005507e8b7ca0a35788b591436d05b4a/torch-2.12.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c75e93173c700bccd6bfcc4a9d19ce242ab6dacd1f1781483027a16239b9e650", size = 87992358, upload-time = "2026-06-17T21:07:40.299Z" },
-    { url = "https://files.pythonhosted.org/packages/30/a7/874a5ca05e8f159211dca7921060f7057acc1adb26431e119fd150623efc/torch-2.12.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fcb61ccd20784b62bdd78ec84238a5cfb383b4994902e03bac95505ab360884c", size = 426386134, upload-time = "2026-06-17T21:07:31.481Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/75/20bb8fe9c1ad6538cce8cd0391b51927ae5af0b17ed1eab44b8824465dc1/torch-2.12.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f4afc8083dff08719edbea346644476e3cec0cf40ebe256be0ee5d5b7c7e8c0d", size = 532268019, upload-time = "2026-06-17T21:05:37.925Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/fa/824ddb662af55b2eabc0dbb7b57c7c0b1bcd93693754a2b8509ec4d16490/torch-2.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:f92609e3b3ce72f25e2eb780d043ced2480c1a86c47c852604fc7a9108648386", size = 122987777, upload-time = "2026-06-17T21:07:09.49Z" },
-    { url = "https://files.pythonhosted.org/packages/63/b7/1b49fe7086ea36839cc80abc43174c43d0ab6f676c0891c871c162f44fe3/torch-2.12.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e9b6f7d2dd66ea87a3ae620069d31335d594c06effb1a383bdd21cfe61e44ece", size = 88010025, upload-time = "2026-06-17T21:07:03.934Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/06/5b44063a6545036dcc680d2d303b137d9176cfb2cc1e1863e3ef94abeb52/torch-2.12.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7973ccd3d2cd35c74449213f7bded199bec6c6247e705cbeda7407af79703d91", size = 426392891, upload-time = "2026-06-17T21:05:52.261Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/dd/c9ce9a4b0eb3c5bb92d9ea56766e2c22559f0b45171149188494edcce80f/torch-2.12.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c64ac4aac16be5e296dcd912305605804b203333c690bf98c55bc09494ee92ad", size = 532272494, upload-time = "2026-06-17T21:06:22.72Z" },
-    { url = "https://files.pythonhosted.org/packages/21/7c/f3a601fc1b1f663ff269bfe553654e638651939aa6563e8daa7167c33098/torch-2.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:f6dc4caf7eb4adb38a2d9f536b51db56310fdd1254e69a2d96767e1367c892b3", size = 122987254, upload-time = "2026-06-17T21:06:33.199Z" },
-    { url = "https://files.pythonhosted.org/packages/e6/8c/b8087556cf81ddd808dbeb34afb8396d7ae7a1694ab489f08b1a0004e7d0/torch-2.12.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:2afbb2bdaa8a95040e733f05492ddf133c3967c9b7ce0abd218d704b6cab437d", size = 88303173, upload-time = "2026-06-17T21:05:06.603Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/07/fe09d1699fbed2afa10ebc692ff2b99d113f2605b6748cea633989e2789a/torch-2.12.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:97eba061fcb042fed191400b15568990073d67eaacaa6ee9b7ca01dd8b790fe9", size = 426404009, upload-time = "2026-06-17T21:04:57.557Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/f7/0ce4f6c1962c60ded7270e0a9eb560fb615c92b89d332cf9e3dff36d5ecc/torch-2.12.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3867b861391701012adb2df93360efb88494dca245a185e3bb7624495cfe3f33", size = 532184292, upload-time = "2026-06-17T21:05:17.526Z" },
-    { url = "https://files.pythonhosted.org/packages/70/db/e384c12aba30320ca92aaaf557456cbcb26f04b4df307728bb8f019f5000/torch-2.12.1-cp314-cp314t-win_amd64.whl", hash = "sha256:dd15595f8fc764cffde8c6361a3beb6ef69a028c851b1b3e70e077f615980d4e", size = 123231142, upload-time = "2026-06-17T21:05:27.061Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/e7/19894fdb51c7dbaf94f5a79bb0871da0992e8e4241e579cb006da46d2e58/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", size = 111178962, upload-time = "2026-07-08T16:05:49.855Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/5c/b1d5de470c54e339b30a92d96683a71bcebd78f5f2a7fc714cd6dc6bbd68/torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045", size = 427198333, upload-time = "2026-07-08T16:05:36.847Z" },
+    { url = "https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4", size = 526555154, upload-time = "2026-07-08T16:05:06.507Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/c9/0bb9d097b03cbaf96bb75b15e867347b8e41bfcdfe0539452d17d9e63993/torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb", size = 122015602, upload-time = "2026-07-08T16:05:45.25Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" },
+    { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/fd/0f2ce40f58aefbdb3392f9acce3c8171940943ae2d661f70558bfa73befb/torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330", size = 122015870, upload-time = "2026-07-08T16:05:27.59Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" },
+    { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" },
+    { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" },
+    { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" },
+    { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" },
+    { url = "https://files.pythonhosted.org/packages/36/76/6dcc7f0c07052102dd36f83cbc5800842a909c8c3fbf1a7f8a5844954de9/torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c", size = 111227066, upload-time = "2026-07-08T16:03:33.6Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" },
+    { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/0c/7d1deb6bce5bc3e6042caf39100ac768eba3b9a098e1dddd16f75bd6489b/torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd", size = 122051871, upload-time = "2026-07-08T16:03:23.521Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/ce/aa8b7f9949d32e0f2f624f342bc3b48112c1b8a130288465938bc83bcbf9/torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1", size = 111537025, upload-time = "2026-07-08T16:02:44.28Z" },
+    { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" },
+    { url = "https://files.pythonhosted.org/packages/56/94/655c91992a882bd5071aa0b5d22a07dbb130d801e872be97c0b627a7c693/torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8", size = 122306773, upload-time = "2026-07-08T16:02:39.832Z" },
 ]
 
 [[package]]
@@ -2928,11 +2965,11 @@ wheels = [
 
 [[package]]
 name = "typing-extensions"
-version = "4.15.0"
+version = "4.16.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
+    { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
 ]
 
 [[package]]
@@ -2949,289 +2986,386 @@ wheels = [
 
 [[package]]
 name = "tzdata"
-version = "2025.2"
+version = "2026.3"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
 ]
 
 [[package]]
 name = "urllib3"
-version = "2.5.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
-]
-
-[[package]]
-name = "werkzeug"
-version = "3.1.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "markupsafe" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
-]
-
-[[package]]
-name = "wheel"
-version = "0.45.1"
+version = "2.7.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
 ]
 
 [[package]]
 name = "wrapt"
-version = "1.17.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" },
-    { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" },
-    { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" },
-    { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" },
-    { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" },
-    { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" },
-    { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" },
-    { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" },
-    { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" },
+version = "2.2.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e1/8b/59781d0fe7b0adfbea37f600857de4be68921e454aeecf1a11bda35cdccc/wrapt-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:055e6fcfaa28e58c6a8c247d48b92be9d56f818b7068aa4f22b15b3343a09931", size = 80556, upload-time = "2026-06-20T23:47:28.473Z" },
+    { url = "https://files.pythonhosted.org/packages/94/dc/66c61aca927230c9cf97a3cb005c803971a1076ff9f7d61085d035c20085/wrapt-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8374eb6b1a58809211e84ff835a182bb17ab2807a5bfef23204c8cff38178a00", size = 81648, upload-time = "2026-06-20T23:47:30.504Z" },
+    { url = "https://files.pythonhosted.org/packages/23/1b/545eee1c18f3af4cf140bb5822b6ef81ebe569df0a63ac109973103a30a5/wrapt-2.2.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:656593bb3f5529f03d27af4136c4d7b11990e470bcbc6fefa5ef218695bece55", size = 152956, upload-time = "2026-06-20T23:47:31.867Z" },
+    { url = "https://files.pythonhosted.org/packages/44/a7/6f42a3d03e44dc612a5dcff324e7366075a7857f0be2d49a8cb8a68279b8/wrapt-2.2.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb00cb7bb22099e2f64b7340fb96113639aa7260c0972af3797ace2297b936c", size = 154771, upload-time = "2026-06-20T23:47:33.352Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/55/4d76175aaa97523c38f1d28f79d18ab41a1b116814158a818bc0eba00571/wrapt-2.2.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e7f10ee0bd53673bfd52b67cbce83336fe6cad90d2377b03baf66491d2bbfb91", size = 149460, upload-time = "2026-06-20T23:47:34.712Z" },
+    { url = "https://files.pythonhosted.org/packages/84/9b/12e23264d8f4735e8483262f95c5a6b03c3665fd2a84bdf99a45b6a2f4ec/wrapt-2.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4402f57c5f0d0579599858ffbdd9bf4e3f0972f51096f2bd6cc7dab6b76ee49e", size = 153648, upload-time = "2026-06-20T23:47:36.092Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/a3/bcd5ec37289dcd85ecd4d15395a6a6063d60bc45ff94a9d77814e1e54d64/wrapt-2.2.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3a4eb7964ff4643d333c84f880bcf554652b2a1050aebc54ae696327f61acfaf", size = 148502, upload-time = "2026-06-20T23:47:37.623Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/be/716d708f607fa70f8a6eb47dff8ee945d5278dfc89ffeeff33039d052e63/wrapt-2.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e542b7c5af91e2123a8aabf19894319d5ec4268d2a9ffd2f239386133fc47746", size = 152238, upload-time = "2026-06-20T23:47:39.118Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/c0/1a48e7e54501274f5d906f18372221b13183b0afbb5b8bb4c7ca0392c0b4/wrapt-2.2.2-cp310-cp310-win32.whl", hash = "sha256:6e7e45b43d3c774d244fe7264378f5a3f0f383bc55a54a9866434e524540110f", size = 77278, upload-time = "2026-06-20T23:47:40.476Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/82/9cd69a1af288fbdedf01a10e3c8a0b6890b08c7f3f96d36a213699dbcd94/wrapt-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:955f1d6e72a352e478de8d8b503abe301c5e139a141b62eb0923bd694995025f", size = 80131, upload-time = "2026-06-20T23:47:41.785Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/73/8db7e27daef37ae70a53ea62bef7fe80cc51a8b5e9e9181a8be6eb9a999c/wrapt-2.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:b89d8d73c82db2bb7e6090b3afd7973f980d24e905cc34394eab60b884b3bf67", size = 79615, upload-time = "2026-06-20T23:47:43.109Z" },
+    { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" },
+    { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" },
+    { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" },
+    { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" },
+    { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" },
+    { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" },
+    { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" },
+    { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" },
+    { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" },
+    { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" },
+    { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" },
+    { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" },
+    { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" },
+    { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" },
+    { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" },
+    { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" },
+    { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" },
+    { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" },
+    { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" },
+    { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" },
+    { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" },
+    { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" },
+    { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" },
+    { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" },
+    { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" },
+    { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" },
+    { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" },
+    { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" },
+    { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" },
+    { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" },
+    { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" },
+    { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" },
+    { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" },
 ]
 
 [[package]]
 name = "xxhash"
-version = "3.6.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845, upload-time = "2025-10-02T14:33:51.573Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807, upload-time = "2025-10-02T14:33:52.964Z" },
-    { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786, upload-time = "2025-10-02T14:33:54.272Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830, upload-time = "2025-10-02T14:33:55.706Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606, upload-time = "2025-10-02T14:33:57.133Z" },
-    { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872, upload-time = "2025-10-02T14:33:58.446Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217, upload-time = "2025-10-02T14:33:59.724Z" },
-    { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139, upload-time = "2025-10-02T14:34:02.041Z" },
-    { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669, upload-time = "2025-10-02T14:34:03.664Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018, upload-time = "2025-10-02T14:34:05.325Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058, upload-time = "2025-10-02T14:34:06.925Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628, upload-time = "2025-10-02T14:34:08.669Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577, upload-time = "2025-10-02T14:34:10.234Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487, upload-time = "2025-10-02T14:34:11.618Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863, upload-time = "2025-10-02T14:34:12.619Z" },
-    { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844, upload-time = "2025-10-02T14:34:14.037Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809, upload-time = "2025-10-02T14:34:15.484Z" },
-    { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665, upload-time = "2025-10-02T14:34:16.541Z" },
-    { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550, upload-time = "2025-10-02T14:34:17.878Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384, upload-time = "2025-10-02T14:34:19.182Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749, upload-time = "2025-10-02T14:34:20.659Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880, upload-time = "2025-10-02T14:34:22.431Z" },
-    { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912, upload-time = "2025-10-02T14:34:23.937Z" },
-    { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654, upload-time = "2025-10-02T14:34:25.644Z" },
-    { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867, upload-time = "2025-10-02T14:34:27.203Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012, upload-time = "2025-10-02T14:34:28.409Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409, upload-time = "2025-10-02T14:34:29.696Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574, upload-time = "2025-10-02T14:34:31.028Z" },
-    { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481, upload-time = "2025-10-02T14:34:32.062Z" },
-    { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861, upload-time = "2025-10-02T14:34:33.555Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" },
-    { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" },
-    { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" },
-    { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" },
-    { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" },
-    { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" },
-    { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" },
-    { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" },
-    { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" },
-    { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" },
-    { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" },
-    { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" },
-    { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" },
-    { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" },
-    { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" },
-    { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" },
-    { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" },
-    { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" },
-    { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" },
-    { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" },
-    { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" },
-    { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" },
-    { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" },
-    { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" },
-    { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" },
-    { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" },
-    { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" },
-    { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" },
-    { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" },
-    { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" },
-    { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" },
-    { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" },
-    { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" },
-    { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481, upload-time = "2025-10-02T14:37:05.869Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565, upload-time = "2025-10-02T14:37:06.966Z" },
+version = "3.8.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/55/97/1a8cebf0a6650417f08a18231590e2515aacd5ce39c3ad8b9e013ebd437d/xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937", size = 34695, upload-time = "2026-07-06T10:43:40.248Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/cf/745b9bc0dd9c341bc074b5fc700db7bbef0f3b69ab21446492296ab37e50/xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c", size = 32376, upload-time = "2026-07-06T10:43:41.97Z" },
+    { url = "https://files.pythonhosted.org/packages/65/a4/8512a901b1d6ad4a9838d1b40385907a879d7e005a5afbec5d39526b69f6/xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780", size = 217470, upload-time = "2026-07-06T10:43:43.572Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/ad/0ffd8094ea29579bb2dc42fa74d08570e9ea3d95db561e6b1105e69b9ca6/xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f", size = 237799, upload-time = "2026-07-06T10:43:45.248Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/90/783c6b3f9336bd07449fe672be32cef6833633936bbfda8d3b23ee18d202/xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce", size = 262587, upload-time = "2026-07-06T10:43:46.733Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/77/ba0316a7c3e661b86830a47ae4987798616ce1b15af8d2a6358e2d89ef60/xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8", size = 238484, upload-time = "2026-07-06T10:43:48.453Z" },
+    { url = "https://files.pythonhosted.org/packages/09/79/33001037c1cba90f4ced38b257161c13452024c0db44208f883e2e47f3fc/xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656", size = 469909, upload-time = "2026-07-06T10:43:50.188Z" },
+    { url = "https://files.pythonhosted.org/packages/45/90/237eded9dd6ae638083294e5a9f77b317aaebd480a330806b39c192a0de1/xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb", size = 217166, upload-time = "2026-07-06T10:43:51.816Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/6a/8cb439dc9920e1468e1c2d69ef77cbeb4be3b1ae9f4b5344c07a2b59af18/xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea", size = 307593, upload-time = "2026-07-06T10:43:53.436Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/c6/c0607d373c8affea92101a3926c4fc8b026bcf8983e05fd58f3a0380ebf8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b", size = 234702, upload-time = "2026-07-06T10:43:55.042Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/cb/f4cfd456624c1f017858168b7ba9443dad810da8aac779a612658450e827/xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f", size = 265749, upload-time = "2026-07-06T10:43:56.749Z" },
+    { url = "https://files.pythonhosted.org/packages/33/f3/9006669c04b01206e21b2177425c649461ba188930a052c2f1728d6ec6a8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef", size = 221992, upload-time = "2026-07-06T10:43:58.12Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/0b/7e6f3eaa05df5e0b6c94aa452b0672801f7031e602081f07fd441aaaaed5/xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8", size = 236899, upload-time = "2026-07-06T10:43:59.562Z" },
+    { url = "https://files.pythonhosted.org/packages/da/cc/bbaee4987f3aab1d7b33bb430bb49e940646160af448b9167431c931126d/xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5", size = 297934, upload-time = "2026-07-06T10:44:01.132Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/97/6bee358660eb8b4f73c00b00b00bc616ebde00e1ab4b67c63486ce360648/xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4", size = 439315, upload-time = "2026-07-06T10:44:02.878Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/50/7e35275f39256bedace0c3cd5be3c72d4ac9d5aecf5e5fdc3530337cd263/xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb", size = 214038, upload-time = "2026-07-06T10:44:04.504Z" },
+    { url = "https://files.pythonhosted.org/packages/59/2d/69d02d096ee50bdf3ef0d208d874f52c71b1aa6906066bce3c52fedb8bc6/xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626", size = 31939, upload-time = "2026-07-06T10:44:06.213Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/1d/e06fca9844919ca91c6587d530cfa1e745830ec73ad38f44f04b25d1bfb7/xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf", size = 32729, upload-time = "2026-07-06T10:44:07.621Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/c2/800648d99039927b5a86d8ae02cd86a556a5ee1678d388216f6b44c8966c/xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3", size = 29215, upload-time = "2026-07-06T10:44:08.916Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/5a/05eaa129555f85476a3e16ff869e95f81a78bbe4647eef9d0229f515a317/xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147", size = 34699, upload-time = "2026-07-06T10:44:10.14Z" },
+    { url = "https://files.pythonhosted.org/packages/80/59/0df1133958b2228929355e022aab1e958c7b2c43e27bf7f59bc9edfa8a54/xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10", size = 32373, upload-time = "2026-07-06T10:44:11.353Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/bf/1cfda5b5e6bf26617812b4a31662ef2220d2ad04e0a55b8ff9eb36e56a5c/xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670", size = 220284, upload-time = "2026-07-06T10:44:12.683Z" },
+    { url = "https://files.pythonhosted.org/packages/70/93/45dc0ad7913b69e5b08bd039236cf628380e4c9cc76a8a4c6625a328e058/xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05", size = 240980, upload-time = "2026-07-06T10:44:14.297Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/02/f28ba7d17f2c1410ee397982c817ab1bd5b2701070c2d2c373539aad000a/xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae", size = 264526, upload-time = "2026-07-06T10:44:16.017Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/d0/f10651cec2c7981b20d693deae6bdfc438427d92be2db4ccabb6181f0021/xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a", size = 241369, upload-time = "2026-07-06T10:44:17.698Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/40/136e0cbaf5db51e191423b1c98643593189f02b6cd90837bf64b19113d70/xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c", size = 473186, upload-time = "2026-07-06T10:44:19.354Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/3f/6aa808a96bdc43dba9a740dec56c744526ee3c0019e32c75e810fa90ae4d/xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60", size = 220092, upload-time = "2026-07-06T10:44:20.956Z" },
+    { url = "https://files.pythonhosted.org/packages/47/28/a8675e78a9ced96dab853416162268e10e05b452e95db7888cf69f58ac5f/xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342", size = 309846, upload-time = "2026-07-06T10:44:22.543Z" },
+    { url = "https://files.pythonhosted.org/packages/89/0f/7fe4d4ef4e69f0033e012396ee2a115886bca7b10b7e45ce398626436bfc/xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723", size = 237659, upload-time = "2026-07-06T10:44:24.135Z" },
+    { url = "https://files.pythonhosted.org/packages/38/8f/83e9e31d4ed57fe963b99cb5b13a23e3e0f0dad1885aa0ebd2a7819dd423/xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a", size = 268737, upload-time = "2026-07-06T10:44:25.875Z" },
+    { url = "https://files.pythonhosted.org/packages/57/79/7e7de46dbe5d1f49afc96a0bc42e6b8df24eae3d6bad6007b99e42f48430/xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937", size = 224955, upload-time = "2026-07-06T10:44:27.777Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/34/b8540839e958d5ef5c6101af6f16032109e7099698ae8edbc8dcefe4d8f4/xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661", size = 239653, upload-time = "2026-07-06T10:44:29.422Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/87/a735d05f7f859354acadabe470ff40e2c46672275f96dcf096a761904def/xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673", size = 300213, upload-time = "2026-07-06T10:44:31.401Z" },
+    { url = "https://files.pythonhosted.org/packages/98/31/3e1cb020237b68117fc212dc5f9753b87f865b4dfee7c1ce62d0836955b5/xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872", size = 442508, upload-time = "2026-07-06T10:44:33.023Z" },
+    { url = "https://files.pythonhosted.org/packages/23/bf/f80090622141cc734b039ce1d15ce3ff6dced375e9680249bf5b9b8c6bf9/xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef", size = 216853, upload-time = "2026-07-06T10:44:34.983Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/a3/60157acecc307b238d3651c2483168e224b48b23a36ae6d6903588341d80/xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792", size = 31936, upload-time = "2026-07-06T10:44:36.542Z" },
+    { url = "https://files.pythonhosted.org/packages/59/5c/ef70c418d878d187b8da56d4cdc06aea6cf5e456b301e96e51e1d2cc8625/xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f", size = 32724, upload-time = "2026-07-06T10:44:38.177Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/25/f008db952cec6b2a26445b456eeed2ebebd65e08e848ebe09ed6ac0634e6/xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d", size = 29212, upload-time = "2026-07-06T10:44:39.577Z" },
+    { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" },
+    { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" },
+    { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" },
+    { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" },
+    { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" },
+    { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" },
+    { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" },
+    { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" },
+    { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" },
+    { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" },
+    { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" },
+    { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" },
+    { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" },
+    { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" },
+    { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" },
+    { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" },
+    { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" },
+    { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" },
+    { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" },
+    { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" },
+    { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" },
+    { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" },
+    { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" },
+    { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" },
+    { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" },
+    { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" },
+    { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" },
+    { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" },
+    { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" },
+    { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" },
+    { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" },
+    { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" },
+    { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" },
+    { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" },
+    { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" },
+    { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" },
+    { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" },
+    { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" },
+    { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" },
+    { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" },
+    { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" },
+    { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" },
+    { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" },
+    { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" },
+    { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" },
+    { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" },
+    { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" },
+    { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" },
+    { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" },
+    { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" },
+    { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" },
+    { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" },
+    { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" },
+    { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" },
+    { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" },
+    { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" },
+    { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" },
+    { url = "https://files.pythonhosted.org/packages/99/e4/4d8040435aeac814fc69ba63621565fbeb19229a138e2568324a26b2a45c/xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12", size = 32687, upload-time = "2026-07-06T10:49:42.803Z" },
+    { url = "https://files.pythonhosted.org/packages/da/6a/975f1f2318c760e5bcec109ed379713ae645d8d856c2a3b9ec5d26857087/xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9", size = 29879, upload-time = "2026-07-06T10:49:45.172Z" },
+    { url = "https://files.pythonhosted.org/packages/08/0b/40a2a55ff52cf635bfdc5eae67a772bec85b4f44c6c737f73f6f528d51d1/xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd", size = 43246, upload-time = "2026-07-06T10:49:47.905Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/6d/56ed2b6b200f26fb474f3fd387d95d0601efcd5bb33430c90c68924bdd77/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02", size = 38202, upload-time = "2026-07-06T10:49:50.59Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/a3/56864d895d1161a9f17502088e9c1fb7c06bde2c2efdde620d22bb7a9c43/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20", size = 34448, upload-time = "2026-07-06T10:49:53.242Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/57/5c6e0908a47f61dca96d01c8ee6fce01ed1050611eb779083ba8758fed81/xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42", size = 32869, upload-time = "2026-07-06T10:49:55.972Z" },
 ]
 
 [[package]]
 name = "yarl"
-version = "1.20.1"
+version = "1.24.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "idna" },
     { name = "multidict" },
     { name = "propcache" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910, upload-time = "2025-06-10T00:42:31.108Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644, upload-time = "2025-06-10T00:42:33.851Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322, upload-time = "2025-06-10T00:42:35.688Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786, upload-time = "2025-06-10T00:42:37.817Z" },
-    { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627, upload-time = "2025-06-10T00:42:39.937Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149, upload-time = "2025-06-10T00:42:42.627Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327, upload-time = "2025-06-10T00:42:44.842Z" },
-    { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054, upload-time = "2025-06-10T00:42:47.149Z" },
-    { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035, upload-time = "2025-06-10T00:42:48.852Z" },
-    { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962, upload-time = "2025-06-10T00:42:51.024Z" },
-    { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399, upload-time = "2025-06-10T00:42:53.007Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649, upload-time = "2025-06-10T00:42:54.964Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563, upload-time = "2025-06-10T00:42:57.28Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609, upload-time = "2025-06-10T00:42:59.055Z" },
-    { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224, upload-time = "2025-06-10T00:43:01.248Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753, upload-time = "2025-06-10T00:43:03.486Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817, upload-time = "2025-06-10T00:43:05.231Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833, upload-time = "2025-06-10T00:43:07.393Z" },
-    { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070, upload-time = "2025-06-10T00:43:09.538Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818, upload-time = "2025-06-10T00:43:11.575Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003, upload-time = "2025-06-10T00:43:14.088Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537, upload-time = "2025-06-10T00:43:16.431Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358, upload-time = "2025-06-10T00:43:18.704Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362, upload-time = "2025-06-10T00:43:20.888Z" },
-    { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979, upload-time = "2025-06-10T00:43:23.169Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274, upload-time = "2025-06-10T00:43:27.111Z" },
-    { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294, upload-time = "2025-06-10T00:43:28.96Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169, upload-time = "2025-06-10T00:43:30.701Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776, upload-time = "2025-06-10T00:43:32.51Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341, upload-time = "2025-06-10T00:43:34.543Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988, upload-time = "2025-06-10T00:43:36.489Z" },
-    { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113, upload-time = "2025-06-10T00:43:38.592Z" },
-    { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485, upload-time = "2025-06-10T00:43:41.038Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686, upload-time = "2025-06-10T00:43:42.692Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" },
-    { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" },
-    { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" },
-    { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" },
-    { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" },
-    { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" },
-    { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" },
-    { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" },
-    { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" },
-    { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" },
-    { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" },
-    { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" },
-    { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" },
-    { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" },
-    { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" },
-    { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" },
-    { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" },
-    { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" },
-    { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" },
-    { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" },
-    { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" },
-    { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" },
-    { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" },
-    { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" },
-    { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" },
-    { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3f/df/f1c7a3de0831cd83194f1a85c5bb431b13f81e6b45079314c86d1c4ef3f2/yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12", size = 129057, upload-time = "2026-05-19T21:27:47.564Z" },
+    { url = "https://files.pythonhosted.org/packages/48/41/7daafb32dd7562bf45b1ce56562e7e1a9146f6479b6456873eb8a3413c40/yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0", size = 91545, upload-time = "2026-05-19T21:27:50.089Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/8f/7b3ec212f1ea0683f55f978e3246bc313c38818664edfc97a9f349a4901e/yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75", size = 91380, upload-time = "2026-05-19T21:27:51.953Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/1b/8bafab7db23b0567ae9db749099b329d91e3b82bc6028b2050ba583e116c/yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727", size = 105957, upload-time = "2026-05-19T21:27:53.98Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/77/21030c2f8d21d21559719beafc772ada2014be933418ed1eaed9cc800e42/yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413", size = 97242, upload-time = "2026-05-19T21:27:55.981Z" },
+    { url = "https://files.pythonhosted.org/packages/50/d8/f9ea63d1b6aa910a866e089d871fff6cbd49caab29b86b35221a62dfa0d5/yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9", size = 114719, upload-time = "2026-05-19T21:27:58.037Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/a3/04e0ee98ac58a249ea7ed75223f5f901ba81a834f0b4921b58e5cec11757/yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2", size = 112140, upload-time = "2026-05-19T21:27:59.618Z" },
+    { url = "https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90", size = 106721, upload-time = "2026-05-19T21:28:02.586Z" },
+    { url = "https://files.pythonhosted.org/packages/65/e7/a52478ebfc66ec989e085c6ae038b9f1bfa4190baa193b133b669c709e2f/yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643", size = 106478, upload-time = "2026-05-19T21:28:04.523Z" },
+    { url = "https://files.pythonhosted.org/packages/04/d8/5508530fea8472542de00013ae280765fc938ee196fc4030c43a498afb36/yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac", size = 105423, upload-time = "2026-05-19T21:28:06.515Z" },
+    { url = "https://files.pythonhosted.org/packages/84/f1/ece28505e9628e8b756e11bb4f28864a17cc33b6b44db4d2aaf0622bf630/yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f", size = 99878, upload-time = "2026-05-19T21:28:08.637Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/52/fb5d34529b46dd84013afcfb30b8d2bc2832ed03d412736f577d604fa393/yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36", size = 114025, upload-time = "2026-05-19T21:28:10.64Z" },
+    { url = "https://files.pythonhosted.org/packages/43/f0/ff9d31aaab024f7a251c0ed308a98ae29bf9f7dc344e78f28b1322431ca2/yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a", size = 105613, upload-time = "2026-05-19T21:28:12.784Z" },
+    { url = "https://files.pythonhosted.org/packages/31/7d/3296fb3f3ecd52bf9ae6c16b0895c1cda7e9170a2083861552b683f70264/yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53", size = 111665, upload-time = "2026-05-19T21:28:14.393Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/74/77aa6ddaca4fbf42e45e675a465c43956dd40702281049975a2aa04eae59/yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342", size = 106914, upload-time = "2026-05-19T21:28:15.893Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/02/7611f22cd1d4ed7373eb7f9ee21fde1046edba2e7c0e514880d760352f48/yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4", size = 92658, upload-time = "2026-05-19T21:28:17.471Z" },
+    { url = "https://files.pythonhosted.org/packages/91/00/671d0add79938127292839ae44506ce2f7fe8909c72d5a931864f128fd0b/yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39", size = 87887, upload-time = "2026-05-19T21:28:19.021Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/c5/1ce244152ff2839645e7cae92f90e7bafcb2c52bea7ff586ac714f14f5df/yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1", size = 128971, upload-time = "2026-05-19T21:28:20.543Z" },
+    { url = "https://files.pythonhosted.org/packages/87/5a/00f36967203ed89cb3acd2c8ed526cc3fed9418eb70ce128160a911c8499/yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c", size = 91507, upload-time = "2026-05-19T21:28:22.556Z" },
+    { url = "https://files.pythonhosted.org/packages/31/d0/1fb0c1cd27288f39f6974da4318c32768d72c9890984541fdf1e2e32a51d/yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d", size = 91343, upload-time = "2026-05-19T21:28:24.092Z" },
+    { url = "https://files.pythonhosted.org/packages/03/ce/d4a646508bed2f8dec6435b40166fe9308dd191262033d3f307b2bbcaecd/yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae", size = 105704, upload-time = "2026-05-19T21:28:25.872Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/07/b3278e82d8bc41485bcf6d856cd0433262593de615b1d3dc43bd3f5bead4/yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a", size = 97281, upload-time = "2026-05-19T21:28:27.352Z" },
+    { url = "https://files.pythonhosted.org/packages/17/5b/4cee6e7c92e487bebe7afc797da0aa54a248ab4e776a68fe369ec29665a5/yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e", size = 114020, upload-time = "2026-05-19T21:28:29.458Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/82/111076571545a7d4f9cca3fbd5c6f40615af58642be09f12328f48022468/yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50", size = 111450, upload-time = "2026-05-19T21:28:31.262Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/ec/08f671f69a444d704aeecebf92af659b67b97a869942411d0a578b08c334/yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003", size = 106384, upload-time = "2026-05-19T21:28:32.856Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/86/ce41e7a7a199340b2330d52b60f25c4074b6636dd0e60b1a80d31a9db042/yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f", size = 106153, upload-time = "2026-05-19T21:28:35.222Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/5d/31be8a729531ab3e55ac3e7e5c800be8c89ea98947f418b2f6ea259fb6ee/yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f", size = 105322, upload-time = "2026-05-19T21:28:36.642Z" },
+    { url = "https://files.pythonhosted.org/packages/47/9b/b57afb22b386ae87ac9940f09878b98d8c333f89113e6fc96fcf4ca9eb64/yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294", size = 99057, upload-time = "2026-05-19T21:28:38.386Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/4f/06348c27c8389256c313e8a57d796808fc0264c915dd5e7cfd3c0e314dc7/yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2", size = 113502, upload-time = "2026-05-19T21:28:40.091Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/1c/284f307b298e4a17b7943b07d9d7ecc4151537f8d137ba51f3bb6c31ca20/yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c", size = 105253, upload-time = "2026-05-19T21:28:41.987Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/bf/0de123bec8619e45c80cbded9085f61b5b4a9eddb8abe6d25d28ee1ec866/yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b", size = 111345, upload-time = "2026-05-19T21:28:43.93Z" },
+    { url = "https://files.pythonhosted.org/packages/90/af/0248eb065e51129d2a9b2436cd1b5c772c19a6b04e5b6a186955671e3319/yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5", size = 106558, upload-time = "2026-05-19T21:28:45.806Z" },
+    { url = "https://files.pythonhosted.org/packages/21/3c/f960d7a65ef97d8ba9b424fb5128796a4bc710fc6df2ddbbd7dfdc3bbd20/yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45", size = 92808, upload-time = "2026-05-19T21:28:48.465Z" },
+    { url = "https://files.pythonhosted.org/packages/03/1a/49fb03750e4de4d2284cd5b885a383133c34eef45bd59631b2bb8b7e81e8/yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122", size = 87610, upload-time = "2026-05-19T21:28:50.07Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957, upload-time = "2026-05-19T21:28:51.695Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164, upload-time = "2026-05-19T21:28:53.242Z" },
+    { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688, upload-time = "2026-05-19T21:28:54.865Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902, upload-time = "2026-05-19T21:28:56.963Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931, upload-time = "2026-05-19T21:28:58.56Z" },
+    { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030, upload-time = "2026-05-19T21:29:00.148Z" },
+    { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392, upload-time = "2026-05-19T21:29:02.155Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487, upload-time = "2026-05-19T21:29:06.491Z" },
+    { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333, upload-time = "2026-05-19T21:29:08.267Z" },
+    { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025, upload-time = "2026-05-19T21:29:10.682Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507, upload-time = "2026-05-19T21:29:12.954Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719, upload-time = "2026-05-19T21:29:14.893Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438, upload-time = "2026-05-19T21:29:16.485Z" },
+    { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901, upload-time = "2026-05-19T21:29:20.014Z" },
+    { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229, upload-time = "2026-05-19T21:29:22.1Z" },
+    { url = "https://files.pythonhosted.org/packages/82/62/fcf0ce677f17e5c471c06311dd25964be38a4c586993632910d2e75278bc/yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536", size = 128978, upload-time = "2026-05-19T21:29:23.83Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/58/8e63299bb71ed61a834121d9d3fe6c9fcf2a6a5d09754ff4f20f2d20baf5/yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607", size = 91733, upload-time = "2026-05-19T21:29:25.375Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/24/16748d5dab6daec8b0ed81ccec639a1cded0f18dcc62a4f696b4fe366c37/yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1", size = 91113, upload-time = "2026-05-19T21:29:26.928Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/66/b63fff7b71211e866624b21432d5943cbb633eb0c2872d9ee3070648f22c/yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986", size = 103899, upload-time = "2026-05-19T21:29:28.842Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/ac/ba1974b8533909636f7733fe86cf677e3619527c3c2fa913e0ea89c48757/yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488", size = 97862, upload-time = "2026-05-19T21:29:31.086Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/a5/123ac993b5c2ba6f554a140305620cb8f150fa543711bbc49be3ec0a65a4/yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b", size = 111060, upload-time = "2026-05-19T21:29:32.657Z" },
+    { url = "https://files.pythonhosted.org/packages/23/37/c472d3af3509688392134a88a825276770a187f1daa4de3f6dc0a327a751/yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592", size = 110613, upload-time = "2026-05-19T21:29:34.379Z" },
+    { url = "https://files.pythonhosted.org/packages/df/88/09c28dad91e662ccfaa1b78f1c57badde74fc9d0b23e74aef644750ecd73/yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617", size = 107012, upload-time = "2026-05-19T21:29:36.216Z" },
+    { url = "https://files.pythonhosted.org/packages/07/ab/9d4f69d571a94f4d112fa7e2e007200f5a54d319f58c82ac7b7baa61f5c6/yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92", size = 105887, upload-time = "2026-05-19T21:29:38.746Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/9a/000b2b66c0d772a499fc531d21dab92dfeb73b640a12eed6ba89f49bb2d0/yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a", size = 103620, upload-time = "2026-05-19T21:29:40.368Z" },
+    { url = "https://files.pythonhosted.org/packages/41/7c/7c1050f73450fbdaa3f0c72017059f00ce5e13366692f3dba25275a1083d/yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44", size = 100599, upload-time = "2026-05-19T21:29:42.66Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/b1/29e5756b3926705f5f6089bd5b9f50a56eaac550da6e260bf713ead44d04/yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a", size = 110604, upload-time = "2026-05-19T21:29:44.632Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/4b/8415bc96e9b150cde942fbac9a8182985e58f40ce5c54c34ed015407d3ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf", size = 105161, upload-time = "2026-05-19T21:29:46.755Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/d4/cde059abfa229553b7298a2eadde2752e723d50aeedaef86ce59da2718ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056", size = 110619, upload-time = "2026-05-19T21:29:48.972Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/2c/d6a6c9a61549f7b6c7e6dc6937d195bcf069582b47b7200dcd0e7b256acf/yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992", size = 107362, upload-time = "2026-05-19T21:29:51Z" },
+    { url = "https://files.pythonhosted.org/packages/92/dd/3ae5fe417e9d1c353a548553326eb9935e76b6b727161563b424cc296df3/yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656", size = 92667, upload-time = "2026-05-19T21:29:52.743Z" },
+    { url = "https://files.pythonhosted.org/packages/10/cc/a7beb239f78f27fca1b053c8e8595e4179c02e62249b4687ec218c370c50/yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461", size = 87069, upload-time = "2026-05-19T21:29:54.442Z" },
+    { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670, upload-time = "2026-05-19T21:29:56.631Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916, upload-time = "2026-05-19T21:29:58.645Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625, upload-time = "2026-05-19T21:30:00.412Z" },
+    { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574, upload-time = "2026-05-19T21:30:02.544Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534, upload-time = "2026-05-19T21:30:04.319Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481, upload-time = "2026-05-19T21:30:05.988Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529, upload-time = "2026-05-19T21:30:07.738Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338, upload-time = "2026-05-19T21:30:09.713Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147, upload-time = "2026-05-19T21:30:11.365Z" },
+    { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272, upload-time = "2026-05-19T21:30:12.978Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962, upload-time = "2026-05-19T21:30:15.001Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063, upload-time = "2026-05-19T21:30:16.683Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438, upload-time = "2026-05-19T21:30:18.769Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458, upload-time = "2026-05-19T21:30:21.024Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589, upload-time = "2026-05-19T21:30:23.412Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424, upload-time = "2026-05-19T21:30:25.425Z" },
+    { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690, upload-time = "2026-05-19T21:30:27.623Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248, upload-time = "2026-05-19T21:30:29.297Z" },
+    { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084, upload-time = "2026-05-19T21:30:31.031Z" },
+    { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272, upload-time = "2026-05-19T21:30:33.062Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497, upload-time = "2026-05-19T21:30:34.842Z" },
+    { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002, upload-time = "2026-05-19T21:30:37.724Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524, upload-time = "2026-05-19T21:30:40.196Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165, upload-time = "2026-05-19T21:30:41.888Z" },
+    { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010, upload-time = "2026-05-19T21:30:43.985Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128, upload-time = "2026-05-19T21:30:46.291Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382, upload-time = "2026-05-19T21:30:48.085Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964, upload-time = "2026-05-19T21:30:49.785Z" },
+    { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204, upload-time = "2026-05-19T21:30:51.862Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510, upload-time = "2026-05-19T21:30:53.62Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584, upload-time = "2026-05-19T21:30:55.962Z" },
+    { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410, upload-time = "2026-05-19T21:30:57.962Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980, upload-time = "2026-05-19T21:30:59.735Z" },
+    { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219, upload-time = "2026-05-19T21:31:01.934Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" },
+]
+
+[[package]]
+name = "zipp"
+version = "4.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" },
 ]
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 5699bd4d536..c4058187023 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 # We keep this pinned to keep clippy and rustfmt in sync between local and CI.
 # Feel free to upgrade to bring in new lints.
 [toolchain]
-channel = "1.94.0"
+channel = "1.97.0"
 components = ["rustfmt", "clippy", "rust-analyzer"]
diff --git a/rust/AGENTS.md b/rust/AGENTS.md
index 6b2729c6692..70a803c6c76 100644
--- a/rust/AGENTS.md
+++ b/rust/AGENTS.md
@@ -17,6 +17,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 - Delete obsolete internal (`pub(crate)` / private) methods in the same PR that introduces their replacements. For public API methods, follow the deprecation path in root AGENTS.md instead.
 - Choose log levels by audience: `debug!` for routine/high-frequency ops, `info!` for infrequent operator-visible state changes, `warn!` for unexpected conditions.
 
+## Concurrency
+
+- The closure passed to `spawn_cpu()` must only consume CPU and return — it must **never** wait on anything: **no channels** (blocking send/recv), **no I/O**, **no locks**, and no `block_on`/`.blocking_*`. The CPU pool can collapse to a single worker in resource-constrained environments (`<= 3` CPUs), so a parked closure can deadlock the whole pool with a silent 0% hang. Keep the waiting in surrounding async code and hand only the pure-CPU work to `spawn_cpu()`. Only dispatch substantial work (rule of thumb: ~100µs+ of CPU); below that the pool overhead outweighs the benefit and the work is better left inline. See the doc comment on `spawn_cpu` for the rationale.
+
 ## API Design
 
 - Use `with_`-prefixed builder methods for optional config (e.g., `MyStruct::new(required).with_option(v)`) — don't create separate constructor variants.
diff --git a/rust/CLAUDE.md b/rust/CLAUDE.md
new file mode 120000
index 00000000000..47dc3e3d863
--- /dev/null
+++ b/rust/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs
index 188a1f4ce2a..b17edacabb2 100644
--- a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs
+++ b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs
@@ -420,12 +420,11 @@ enum InstructionSet {
     Scalar,
 }
 
-/// Internal 8-wide bitpacker implementation.
+/// 8-wide bitpacker implementation.
 ///
-/// One block contains 256 integers. This stays private to avoid exposing a new
-/// block-size choice through the public Lance bitpacking API.
+/// One block contains 256 integers.
 #[derive(Clone, Copy)]
-pub(crate) struct BitPacker8x(InstructionSet);
+pub struct BitPacker8x(InstructionSet);
 
 impl BitPacker8x {
     #[cfg(target_arch = "x86_64")]
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs
index c287a29da0b..80803e50ec8 100644
--- a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs
+++ b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs
@@ -20,6 +20,7 @@ mod bitpacker4x;
 mod bitpacker8x;
 
 pub use bitpacker4x::BitPacker4x;
+pub use bitpacker8x::BitPacker8x;
 
 pub(crate) trait Available {
     fn available() -> bool;
diff --git a/rust/compression/bitpacking/src/lib.rs b/rust/compression/bitpacking/src/lib.rs
index f0e25e37e8c..c6aa6d75ad0 100644
--- a/rust/compression/bitpacking/src/lib.rs
+++ b/rust/compression/bitpacking/src/lib.rs
@@ -18,7 +18,7 @@ use core::mem::size_of;
 
 mod bitpacker_internal;
 
-pub use bitpacker_internal::{BitPacker, BitPacker4x};
+pub use bitpacker_internal::{BitPacker, BitPacker4x, BitPacker8x};
 
 pub const FL_ORDER: [usize; 8] = [0, 4, 2, 6, 1, 5, 3, 7];
 
diff --git a/rust/compression/fsst/Cargo.toml b/rust/compression/fsst/Cargo.toml
index da5d8f01d04..7056896e3b9 100644
--- a/rust/compression/fsst/Cargo.toml
+++ b/rust/compression/fsst/Cargo.toml
@@ -16,7 +16,6 @@ arrow-array.workspace = true
 rand.workspace = true
 
 [dev-dependencies]
-arrow-array.workspace = true
 test-log.workspace = true
 tokio.workspace = true
 
diff --git a/rust/compression/fsst/examples/benchmark.rs b/rust/compression/fsst/examples/benchmark.rs
index c442243e112..f71abeefedd 100644
--- a/rust/compression/fsst/examples/benchmark.rs
+++ b/rust/compression/fsst/examples/benchmark.rs
@@ -56,7 +56,10 @@ fn benchmark(file_path: &str) {
     let mut decompression_out_bufs = vec![];
     let mut decompression_out_offsets_bufs = vec![];
     for _ in 0..TEST_NUM {
-        let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 3];
+        // `decompress` requires the output buffer to be at least 8x the compressed input (a 1-byte
+        // code can expand to an 8-byte symbol). The compressed buffer is at most `BUFFER_SIZE`, so
+        // `BUFFER_SIZE * 8` is a safe upper bound.
+        let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 8];
         let this_decom_out_offsets_buf = vec![0i32; BUFFER_SIZE * 3];
         decompression_out_bufs.push(this_decom_out_buf);
         decompression_out_offsets_bufs.push(this_decom_out_offsets_buf);
diff --git a/rust/compression/fsst/src/fsst.rs b/rust/compression/fsst/src/fsst.rs
index 0a2bf1d03d7..d00a6ed806b 100644
--- a/rust/compression/fsst/src/fsst.rs
+++ b/rust/compression/fsst/src/fsst.rs
@@ -57,6 +57,12 @@ use std::ptr;
 
 #[inline]
 fn fsst_unaligned_load_unchecked(v: *const u8) -> u64 {
+    // SAFETY: the caller must guarantee that `v` points to at least 8 readable bytes. All callers
+    // uphold this: `compress_bulk` loads from a 520-byte stack buffer at an offset < 511 (leaving
+    // >= 8 bytes), `build_symbol_table` guards the load with `word.len() > 7 && curr < word.len() - 7`,
+    // `find_longest_symbol_from_char_slice` copies into a stack `[u8; 8]` before loading, and
+    // `FsstDecoder::init` reads symbols from a `symbol_table` buffer already validated to be
+    // exactly `FSST_SYMBOL_TABLE_SIZE` bytes.
     unsafe { ptr::read_unaligned(v as *const u64) }
 }
 
@@ -812,12 +818,31 @@ fn decompress_bulk<T: OffsetSizeTrait>(
 ) -> io::Result<()> {
     let symbols = decoder.symbols;
     let lens = decoder.lens;
+    // SAFETY invariant shared by every `unsafe` block in this closure:
+    // - `out` is sized to at least 8x `compressed_strs` (checked in `FsstDecoder::init`, which the
+    //   sole public entry point always runs before reaching this function). Each code advances
+    //   `out_curr` by `lens[code]`, which is 1..=8 for a well-formed symbol table, and each
+    //   consumed input byte yields at most 8 output bytes, so `out_curr + 8 <= out.len()` at every
+    //   8-byte write, including the final one. This is why we can `write_unaligned` a full 8-byte
+    //   word per code and advance by only the length.
+    //   NOTE: `lens` is loaded verbatim from the (untrusted) symbol table and is NOT re-validated
+    //   to be <= 8 on decode, and offsets (below) are likewise trusted. A corrupted table or
+    //   offset buffer can violate these bounds; callers must supply structures produced by
+    //   `compress` (or otherwise trusted). Hardening the decoder against corrupt input is a
+    //   separate concern, not addressed here.
+    // - The only unchecked read is `read_unaligned::<u32>`, gated by `in_curr + 4 <= in_end`; the
+    //   scalar paths use bounds-checked indexing. `in_end` is a caller-provided offset into
+    //   `compressed_strs`; the read is sound only if `in_end <= compressed_strs.len()`, which is a
+    //   trusted precondition (holds for encoder-produced offsets; not validated here).
     let mut decompress = |mut in_curr: usize, in_end: usize, out_curr: &mut usize| {
         // Do SIMD operation here by 4 bytes
         while in_curr + 4 <= in_end {
             let next_block;
             let mut code;
             let mut len;
+            // SAFETY: the loop guard proves `in_curr + 4 <= in_end`. Per the closure-level
+            // invariant, `in_end <= compressed_strs.len()` is a trusted precondition (not checked
+            // here), so the 4-byte read is in bounds for well-formed input.
             unsafe {
                 next_block =
                     ptr::read_unaligned(compressed_strs.as_ptr().add(in_curr) as *const u32);
@@ -983,6 +1008,10 @@ fn decompress_bulk<T: OffsetSizeTrait>(
         if in_curr < in_end {
             // last code cannot be an escape code
             let code = compressed_strs[in_curr] as usize;
+            // SAFETY: see the closure-level invariant. This is the final write and has no
+            // subsequent write to cover its slack, so it is the tightest case: `out_curr` is at
+            // most 8*(consumed_input_bytes - 1), and the 8-byte store lands within `out.len()`
+            // precisely because the caller sized `out` to 8x the input.
             unsafe {
                 let src = symbols[code];
                 ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src);
@@ -1188,8 +1217,19 @@ impl<T: OffsetSizeTrait> FsstDecoder<T> {
         }
 
         self.decoder_switch_on = (st_info & (1 << 24)) != 0;
-        // when decoder_switch_on is true, we make sure the out_buf is at least 3 times the size of the in_buf,
-        if self.decoder_switch_on && in_buf.len() * 3 > out_buf.len() {
+        // A single 1-byte code can decode to a symbol of up to MAX_SYMBOL_LENGTH (8) bytes, so the
+        // decoded output can be up to 8x the input. `decompress_bulk` also relies on this bound: it
+        // writes a full 8-byte word per code (advancing only by the symbol length), so the output
+        // buffer must be large enough that even the final write stays in bounds. Require out_buf to
+        // be at least 8x in_buf. `checked_mul` guards against `in_buf.len() * 8` wrapping on 32-bit
+        // targets (an input >= 512 MiB would otherwise bypass the check); treat overflow as too
+        // small.
+        if self.decoder_switch_on
+            && in_buf
+                .len()
+                .checked_mul(8)
+                .is_none_or(|needed| needed > out_buf.len())
+        {
             return Err(io::Error::new(
                 io::ErrorKind::InvalidInput,
                 "output buffer too small for FSST decoder",
@@ -1288,8 +1328,10 @@ pub fn compress<T: OffsetSizeTrait>(
 // the following 32 bits after FSST_MAGIC contains information about FSST encoding, such as decoder_switch_on, suffix_lim, terminator, n_symbols
 // when the decoder_switch_on is off in the in_buf header, `decompress` first make sure the out_buf is at least the same size as the in_buf, then simply copy the
 // input data to the output
-// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 3 times the size of the in_buf, then start decoding the
-// data using the symbol table
+// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 8 times the size of the in_buf, then start decoding the
+// data using the symbol table. The 8x bound is required for correctness: a 1-byte code can expand to
+// an 8-byte symbol, and the decode loop writes a full 8-byte word per code, so a smaller buffer can
+// be written out of bounds.
 // the out_offsets_buf should be at least the same size as the in_offsets_buf, otherwise an error is returned
 // the symbol_table is the same symbol table created by `compression`
 pub fn decompress<T: OffsetSizeTrait>(
@@ -1644,4 +1686,59 @@ But exactly how the acquaintance and friendship came about, we cannot say.";
             );
         }
     }
+
+    // Build a genuinely FSST-compressed (decoder_switch_on) buffer to exercise the decode-side
+    // output-buffer size contract. Returns (symbol_table, compressed_bytes, compressed_offsets).
+    fn compress_paragraph() -> ([u8; FSST_SYMBOL_TABLE_SIZE], Vec<u8>, Vec<i32>) {
+        let test_input = TEST_PARAGRAPH.repeat((1024 * 1024) / TEST_PARAGRAPH.len());
+        let lines_vec = test_input.lines().collect::<Vec<&str>>();
+        let string_array = StringArray::from(lines_vec);
+        let mut compress_output_buf: Vec<u8> = vec![0; string_array.value_data().len()];
+        let mut compress_offset_buf: Vec<i32> = vec![0; string_array.value_offsets().len()];
+        let mut symbol_table = [0; FSST_SYMBOL_TABLE_SIZE];
+        compress(
+            symbol_table.as_mut(),
+            string_array.value_data(),
+            string_array.value_offsets(),
+            &mut compress_output_buf,
+            &mut compress_offset_buf,
+        )
+        .unwrap();
+        (symbol_table, compress_output_buf, compress_offset_buf)
+    }
+
+    // The decoder writes a full 8-byte word per code, so the output buffer must be at least 8x the
+    // compressed input. A buffer sized below 8x must be rejected rather than written out of bounds.
+    #[test_log::test(tokio::test)]
+    async fn test_decompress_rejects_undersized_output_buffer() {
+        let (symbol_table, compressed, compressed_offsets) = compress_paragraph();
+        // Sanity check: this input actually engaged FSST compression (decoder_switch_on).
+        let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap());
+        assert!(st_info & (1 << 24) != 0, "expected decoder_switch_on input");
+
+        // One byte short of the 8x requirement must be rejected.
+        let mut too_small = vec![0u8; compressed.len() * 8 - 1];
+        let mut out_offsets = vec![0i32; compressed_offsets.len()];
+        let err = decompress(
+            &symbol_table,
+            &compressed,
+            &compressed_offsets,
+            &mut too_small,
+            &mut out_offsets,
+        )
+        .unwrap_err();
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+
+        // Exactly 8x is the tight bound and must succeed.
+        let mut exact = vec![0u8; compressed.len() * 8];
+        let mut out_offsets = vec![0i32; compressed_offsets.len()];
+        decompress(
+            &symbol_table,
+            &compressed,
+            &compressed_offsets,
+            &mut exact,
+            &mut out_offsets,
+        )
+        .unwrap();
+    }
 }
diff --git a/rust/examples/src/hnsw.rs b/rust/examples/src/hnsw.rs
index 5be0debf6e1..52c15bd0851 100644
--- a/rust/examples/src/hnsw.rs
+++ b/rust/examples/src/hnsw.rs
@@ -129,6 +129,7 @@ async fn main() {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let results: HashSet<u32> = hnsw
             .search_basic(q.clone(), k, &params, None, vector_store.as_ref())
diff --git a/rust/lance-arrow/Cargo.toml b/rust/lance-arrow/Cargo.toml
index afbc9c26ed3..21513e638d8 100644
--- a/rust/lance-arrow/Cargo.toml
+++ b/rust/lance-arrow/Cargo.toml
@@ -21,6 +21,7 @@ arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
+bytemuck = { workspace = true }
 bytes = { workspace = true }
 futures = { workspace = true }
 half = { workspace = true }
diff --git a/rust/lance-arrow/src/bfloat16.rs b/rust/lance-arrow/src/bfloat16.rs
index 2f59de51317..74c7f259a40 100644
--- a/rust/lance-arrow/src/bfloat16.rs
+++ b/rust/lance-arrow/src/bfloat16.rs
@@ -7,7 +7,7 @@ use std::fmt::Formatter;
 use std::slice;
 
 use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder};
-use arrow_buffer::MutableBuffer;
+use arrow_buffer::{Buffer, MutableBuffer};
 use arrow_data::ArrayData;
 use arrow_schema::{ArrowError, DataType, Field as ArrowField};
 use half::bf16;
@@ -164,19 +164,18 @@ impl FromIterator<bf16> for BFloat16Array {
 
 impl From<Vec<bf16>> for BFloat16Array {
     fn from(data: Vec<bf16>) -> Self {
-        let mut buffer = MutableBuffer::with_capacity(data.len() * 2);
-
-        // Write each value's little-endian bytes straight into the buffer. Going
-        // through an intermediate `Vec` per element would allocate once per value.
-        for val in &data {
-            buffer.extend_from_slice(&val.to_bits().to_le_bytes());
-        }
-
+        let len = data.len();
+        // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives
+        // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place —
+        // no per-element copy or heap alloc. The crate-root `compile_error!`
+        // pins `target_endian = "little"`, so the resulting bytes match the
+        // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere.
+        let raw: Vec<u16> = bytemuck::cast_vec(data);
         let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
-            .len(data.len())
-            .add_buffer(buffer.into());
-        // SAFETY: the buffer contains exactly `2 * data.len()` bytes — each
-        // `bf16` writes its two little-endian bytes once — matching the
+            .len(len)
+            .add_buffer(Buffer::from_vec(raw));
+        // SAFETY: the value buffer contains exactly `2 * len` bytes — one
+        // `u16` per element after the layout-compatible cast — matching the
         // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so
         // every element is logically valid.
         let array_data = unsafe { array_data.build_unchecked() };
@@ -284,9 +283,10 @@ impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
     /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape
     ///   used by [`BFloat16Array`]). Asserted at entry.
     /// - The value buffer must be at least 2-byte aligned. Lance's in-tree
-    ///   constructors always satisfy this (every value buffer goes through
-    ///   `MutableBuffer`, which is aligned to arrow-buffer's `ALIGNMENT`
-    ///   constant — ≥32 bytes on every supported target). Externally-built
+    ///   constructors always satisfy this: value buffers are built either via
+    ///   `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32
+    ///   bytes) or via `Buffer::from_vec::<u16>` (aligned to `align_of::<u16>()`
+    ///   == 2); both meet `bf16`'s 2-byte requirement. Externally-built
     ///   `FixedSizeBinaryArray`s arriving via FFI, IPC, or
     ///   `Buffer::from_custom_allocation` are not required by arrow-rs to be
     ///   aligned beyond a single byte; passing one to this method violates the
@@ -329,8 +329,8 @@ impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
         //   (arrow-data `data.rs`), so arrow-rs alone does not guarantee
         //   2-byte alignment. Lance's in-tree construction paths build value
         //   buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant,
-        //   ≥32 bytes on every supported target), which trivially satisfies
-        //   `bf16`'s 2-byte requirement.
+        //   ≥32 bytes) or `Buffer::from_vec::<u16>` (2-byte aligned), both of
+        //   which satisfy `bf16`'s 2-byte requirement.
         // - The returned slice borrows from `self`; the underlying ref-counted,
         //   immutable Arrow buffer cannot be mutated or freed for the slice's
         //   lifetime.
@@ -361,6 +361,16 @@ mod tests {
         assert_eq!(array, array2);
         assert_eq!(array.len(), 3);
 
+        // Pin the raw little-endian bytes emitted by `From<Vec<bf16>>` (rewritten to
+        // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order
+        // regression is caught directly rather than only through Debug formatting.
+        // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040.
+        let inner = array2.clone().into_inner();
+        let raw_bytes: Vec<u8> = (0..inner.len())
+            .flat_map(|i| inner.value(i).to_vec())
+            .collect();
+        assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]);
+
         let expected_fmt = "BFloat16Array\n[\n  1.0,\n  2.0,\n  3.0,\n]";
         assert_eq!(expected_fmt, format!("{:?}", array));
 
diff --git a/rust/lance-arrow/src/lib.rs b/rust/lance-arrow/src/lib.rs
index a55b42cb6c0..2ac962aa1aa 100644
--- a/rust/lance-arrow/src/lib.rs
+++ b/rust/lance-arrow/src/lib.rs
@@ -1022,13 +1022,7 @@ fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc<dyn Array> {
 fn normalize_validity(
     validity: Option<&arrow_buffer::NullBuffer>,
 ) -> Option<&arrow_buffer::NullBuffer> {
-    validity.and_then(|v| {
-        if v.null_count() == v.len() {
-            None
-        } else {
-            Some(v)
-        }
-    })
+    validity.filter(|v| v.null_count() != v.len())
 }
 
 /// Helper function to merge validity buffers from two struct arrays
@@ -1385,9 +1379,12 @@ fn merge_with_schema(
                         );
                         let merged_validity =
                             merge_struct_validity(left_list.nulls(), right_list.nulls());
+                        // `trimmed_values` starts at the first used value, so offsets
+                        // must be shifted to match or `ListArray::new` panics when the
+                        // input list was sliced (e.g. from a filtered batch).
                         let merged_list = ListArray::new(
                             child_field.clone(),
-                            left_list.offsets().clone(),
+                            left_list.trimmed_offsets(),
                             merged_values,
                             merged_validity,
                         );
@@ -1412,7 +1409,7 @@ fn merge_with_schema(
                             merge_struct_validity(left_list.nulls(), right_list.nulls());
                         let merged_list = LargeListArray::new(
                             child_field.clone(),
-                            left_list.offsets().clone(),
+                            left_list.trimmed_offsets(),
                             merged_values,
                             merged_validity,
                         );
@@ -2380,6 +2377,118 @@ mod tests {
         assert_eq!(merged_array.len(), 2);
     }
 
+    #[test]
+    fn test_merge_with_schema_sliced_list_struct() {
+        test_merge_with_schema_sliced_list_struct_generic::<i32>();
+    }
+
+    #[test]
+    fn test_merge_with_schema_sliced_large_list_struct() {
+        test_merge_with_schema_sliced_list_struct_generic::<i64>();
+    }
+
+    // Regression for #6580: merge_with_schema panicked when the left list was a
+    // sliced view whose offsets did not start at zero (common after a filtered
+    // scan). Cloning those offsets alongside `trimmed_values` produced offsets
+    // larger than the trimmed child, panicking in `(Large)ListArray::new`.
+    fn test_merge_with_schema_sliced_list_struct_generic<O: OffsetSizeTrait>() {
+        let make_list_dtype = |item_field: Arc<Field>| {
+            if O::IS_LARGE {
+                DataType::LargeList(item_field)
+            } else {
+                DataType::List(item_field)
+            }
+        };
+
+        // Build a List<Struct> with two rows of 5 items each, then slice away
+        // the first row so the remaining list's offsets start at 5, not 0.
+        let struct_fields_a = Fields::from(vec![Field::new("a", DataType::Int32, true)]);
+        let left_values = Arc::new(StructArray::new(
+            struct_fields_a.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..10)) as ArrayRef],
+            None,
+        ));
+        let full_list = GenericListArray::<O>::new(
+            Arc::new(Field::new("item", DataType::Struct(struct_fields_a), true)),
+            OffsetBuffer::<O>::from_lengths([5, 5]),
+            left_values,
+            None,
+        );
+        let sliced_left = full_list.slice(1, 1);
+        assert_eq!(sliced_left.offsets()[0].as_usize(), 5);
+        assert_eq!(sliced_left.offsets()[1].as_usize(), 10);
+
+        let struct_fields_b = Fields::from(vec![Field::new("b", DataType::Int32, true)]);
+        let right_values = Arc::new(StructArray::new(
+            struct_fields_b.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(100..105)) as ArrayRef],
+            None,
+        ));
+        let right_list = GenericListArray::<O>::new(
+            Arc::new(Field::new("item", DataType::Struct(struct_fields_b), true)),
+            OffsetBuffer::<O>::from_lengths([5]),
+            right_values,
+            None,
+        );
+
+        let target_item_field = Arc::new(Field::new(
+            "item",
+            DataType::Struct(Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Int32, true),
+            ])),
+            true,
+        ));
+        let target_fields = Fields::from(vec![Field::new(
+            "items",
+            make_list_dtype(target_item_field),
+            true,
+        )]);
+
+        let left_batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                "items",
+                sliced_left.data_type().clone(),
+                true,
+            )])),
+            vec![Arc::new(sliced_left) as ArrayRef],
+        )
+        .unwrap();
+        let right_batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                "items",
+                right_list.data_type().clone(),
+                true,
+            )])),
+            vec![Arc::new(right_list) as ArrayRef],
+        )
+        .unwrap();
+
+        let merged = left_batch
+            .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec()))
+            .unwrap();
+
+        let merged_list = merged
+            .column_by_name("items")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<GenericListArray<O>>()
+            .unwrap();
+        assert_eq!(merged_list.len(), 1);
+        assert_eq!(merged_list.value_length(0).as_usize(), 5);
+        let merged_struct = merged_list.values().as_struct();
+        assert_eq!(merged_struct.num_columns(), 2);
+        let a = merged_struct
+            .column_by_name("a")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        // After shifting offsets to zero, values 5..10 should be first.
+        let a_vals: Vec<i32> = a.iter().map(|v| v.unwrap()).collect();
+        assert_eq!(a_vals, vec![5, 6, 7, 8, 9]);
+    }
+
     #[test]
     fn test_project_by_schema_list_struct_reorder() {
         // Test that project_by_schema correctly reorders fields inside List<Struct>
diff --git a/rust/lance-arrow/src/list.rs b/rust/lance-arrow/src/list.rs
index 0c24fc579da..06b0fc592cf 100644
--- a/rust/lance-arrow/src/list.rs
+++ b/rust/lance-arrow/src/list.rs
@@ -23,6 +23,16 @@ pub trait ListArrayExt {
     /// behaves similarly to `values()` except it slices the array so that it starts at
     /// the first list offset and ends at the last list offset.
     fn trimmed_values(&self) -> Arc<dyn Array>;
+    /// The offset type of the underlying list array.
+    type Offset: OffsetSizeTrait;
+    /// Returns offsets shifted so the first offset is zero, matching
+    /// [`Self::trimmed_values`].
+    ///
+    /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the
+    /// original values buffer, so combining them with trimmed values produces
+    /// offsets that exceed the values length. Use this together with
+    /// `trimmed_values` when constructing a new list array.
+    fn trimmed_offsets(&self) -> OffsetBuffer<Self::Offset>;
 }
 
 impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize> {
@@ -90,6 +100,20 @@ impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize>
             .unwrap_or(0);
         self.values().slice(first_value, last_value - first_value)
     }
+
+    type Offset = OffsetSize;
+
+    fn trimmed_offsets(&self) -> OffsetBuffer<OffsetSize> {
+        let offsets = self.offsets();
+        let Some(&first) = offsets.first() else {
+            return offsets.clone();
+        };
+        if first == OffsetSize::zero() {
+            return offsets.clone();
+        }
+        let shifted: Vec<OffsetSize> = offsets.iter().map(|&o| o - first).collect();
+        OffsetBuffer::new(ScalarBuffer::from(shifted))
+    }
 }
 
 #[cfg(test)]
diff --git a/rust/lance-core/Cargo.toml b/rust/lance-core/Cargo.toml
index 7f956c70430..2f6183be8a1 100644
--- a/rust/lance-core/Cargo.toml
+++ b/rust/lance-core/Cargo.toml
@@ -55,6 +55,10 @@ proptest.workspace = true
 rstest.workspace = true
 
 [features]
+# Capture Rust backtraces in error types. When disabled (the default),
+# the backtrace field is zero-sized with no overhead. At runtime, capture
+# is still gated by RUST_BACKTRACE=1.
+backtrace = []
 datafusion = ["dep:datafusion-common", "dep:datafusion-sql"]
 
 [lints]
diff --git a/rust/lance-core/src/cache/mod.rs b/rust/lance-core/src/cache/mod.rs
index 4f93f261bea..cd7476011bf 100644
--- a/rust/lance-core/src/cache/mod.rs
+++ b/rust/lance-core/src/cache/mod.rs
@@ -595,8 +595,10 @@ impl CacheStats {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::Error;
     use std::collections::{BTreeSet, HashMap};
     use std::marker::PhantomData;
+    use std::sync::atomic::AtomicUsize;
 
     struct TestKey<T: 'static> {
         key: String,
@@ -919,6 +921,37 @@ mod tests {
         assert_eq!(cache.stats().await.hits, 1);
     }
 
+    #[tokio::test]
+    async fn test_cache_coalesces_concurrent_loader_errors() {
+        let cache = LanceCache::with_capacity(1000);
+        let barrier = Arc::new(tokio::sync::Barrier::new(5));
+        let loader_calls = Arc::new(AtomicUsize::new(0));
+
+        let results = futures::future::join_all((0..5).map(|_| {
+            let cache = cache.clone();
+            let barrier = barrier.clone();
+            let loader_calls = loader_calls.clone();
+            async move {
+                barrier.wait().await;
+                cache
+                    .get_or_insert_with_key(TestKey::<Vec<i32>>::new("error"), || async move {
+                        loader_calls.fetch_add(1, Ordering::Relaxed);
+                        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+                        Err(Error::timeout("metadata read timed out"))
+                    })
+                    .await
+            }
+        }))
+        .await;
+
+        assert_eq!(loader_calls.load(Ordering::Relaxed), 1);
+        assert!(
+            results
+                .iter()
+                .all(|result| matches!(result, Err(Error::Timeout { .. })))
+        );
+    }
+
     #[tokio::test]
     async fn test_custom_backend() {
         use async_trait::async_trait;
diff --git a/rust/lance-core/src/cache/moka.rs b/rust/lance-core/src/cache/moka.rs
index fd86b064f6b..024d11cdce8 100644
--- a/rust/lance-core/src/cache/moka.rs
+++ b/rust/lance-core/src/cache/moka.rs
@@ -9,6 +9,7 @@ use async_trait::async_trait;
 use futures::Future;
 
 use crate::Result;
+use crate::error::CloneableError;
 
 use super::CacheCodec;
 use super::backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey};
@@ -88,37 +89,25 @@ impl CacheBackend for MokaCacheBackend {
         loader: Pin<Box<dyn Future<Output = Result<(CacheEntry, usize)>> + Send + 'a>>,
         _codec: Option<CacheCodec>,
     ) -> Result<(CacheEntry, bool)> {
-        // Use moka's built-in dedup: optionally_get_with runs the init future
-        // at most once per key, even under concurrent access.
-        let (error_tx, error_rx) = tokio::sync::oneshot::channel();
-
         // Track whether the loader actually ran (= cache miss).
         let was_miss = Arc::new(AtomicBool::new(false));
         let was_miss_clone = was_miss.clone();
 
         let init = async move {
             was_miss_clone.store(true, Ordering::Relaxed);
-            match loader.await {
-                Ok((entry, size_bytes)) => Some(MokaCacheEntry { entry, size_bytes }),
-                Err(e) => {
-                    let _ = error_tx.send(e);
-                    None
-                }
-            }
+            loader
+                .await
+                .map(|(entry, size_bytes)| MokaCacheEntry { entry, size_bytes })
+                .map_err(CloneableError)
         };
 
         let owned_key = key.clone();
-        match self.cache.optionally_get_with(owned_key, init).await {
-            Some(record) => {
+        match self.cache.try_get_with(owned_key, init).await {
+            Ok(record) => {
                 let was_cached = !was_miss.load(Ordering::Relaxed);
                 Ok((record.entry, was_cached))
             }
-            None => match error_rx.await {
-                Ok(err) => Err(err),
-                Err(_) => Err(crate::Error::internal(
-                    "Failed to retrieve error from cache loader",
-                )),
-            },
+            Err(error) => Err(Arc::unwrap_or_clone(error).0),
         }
     }
 
diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs
index 9f06d421949..d5eb89dccb0 100644
--- a/rust/lance-core/src/datatypes/field.rs
+++ b/rust/lance-core/src/datatypes/field.rs
@@ -58,6 +58,8 @@ pub const LANCE_UNENFORCED_CLUSTERING_KEY_POSITION: &str =
 /// The value should be non-negative i32 value. Any negative value will be seen as -1.
 pub const LANCE_FIELD_ID_KEY: &str = "lance:field_id";
 
+const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
+
 fn has_blob_v2_extension(field: &ArrowField) -> bool {
     field
         .metadata()
@@ -269,24 +271,15 @@ impl Field {
     }
 
     pub fn apply_projection(&self, projection: &Projection) -> Option<Self> {
-        // Map fields encode their physical layout as a single child entries
-        // struct (`Struct<key, value>`) whose presence is required for the
-        // parent to be readable — we never want to filter into that subtree.
-        // But the parent field itself is still subject to selection: if the
-        // caller didn't ask for this Map column, drop it like any other
-        // non-selected leaf. Without this early return the unconditional
-        // children clone would keep `children.is_empty() == false` forever
-        // and every Map column in the schema would survive every projection,
-        // pulling tens-of-bytes-per-row of unrelated data through downstream
-        // operators (notably `SortExec` in scalar-index training, where it
-        // was responsible for >100 GiB external-sort spills on real-world
-        // tables).
-        if self.logical_type.is_map() && !projection.contains_field_id(self.id) {
+        // Maps and blob descriptors are atomic physical layouts. Map children
+        // must remain together, while projected blob descriptor children may
+        // have synthetic IDs that cannot be selected independently.
+        let is_atomic_layout = self.logical_type.is_map() || self.is_blob();
+        if is_atomic_layout && !projection.contains_field_id(self.id) {
             return None;
         }
 
-        let children = if self.logical_type.is_map() {
-            // Map field is selected: keep all children intact.
+        let children = if is_atomic_layout {
             self.children.clone()
         } else {
             self.children
@@ -549,6 +542,20 @@ impl Field {
             .get(ARROW_EXT_NAME_KEY)
             .map(|name| name == BLOB_V2_EXT_NAME)
             .unwrap_or(false)
+            || self.is_blob_v2_descriptor()
+    }
+
+    fn is_blob_v2_descriptor(&self) -> bool {
+        self.metadata.contains_key(BLOB_META_KEY)
+            && self.logical_type == BLOB_V2_DESC_LANCE_FIELD.logical_type
+            && self.children.len() == BLOB_V2_DESC_LANCE_FIELD.children.len()
+            && self
+                .children
+                .iter()
+                .zip(BLOB_V2_DESC_LANCE_FIELD.children.iter())
+                .all(|(child, expected)| {
+                    child.name == expected.name && child.data_type() == expected.data_type()
+                })
     }
 
     // Blob columns intentionally have two schema representations:
@@ -575,6 +582,31 @@ impl Field {
         }
     }
 
+    /// Convert a blob field to the materialized binary payload view.
+    ///
+    /// The field keeps its name and id but uses `LargeBinary` with no children.
+    /// Blob v2 fields retain their extension marker internally so scan planning
+    /// can recognize the binary view before exposing a plain Arrow binary field.
+    pub fn binary_blob_mut(&mut self) {
+        if !self.is_blob() {
+            return;
+        }
+        let is_blob_v2 = self.is_blob_v2();
+
+        self.logical_type = LogicalType::try_from(&DataType::LargeBinary)
+            .expect("LargeBinary is always a valid logical type");
+        self.children.clear();
+        self.encoding = Some(Encoding::VarBinary);
+        if is_blob_v2 {
+            self.metadata.remove(BLOB_META_KEY);
+            for key in PACKED_KEYS {
+                self.metadata.remove(key);
+            }
+            self.metadata
+                .insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string());
+        }
+    }
+
     /// Convert blob v2 fields in this field tree to their descriptor view.
     pub fn unload_blobs_recursive(&mut self) {
         if self.is_blob_v2() {
@@ -812,6 +844,13 @@ impl Field {
         }
 
         if self.is_blob() != other.is_blob() {
+            if ignore_types {
+                return Ok(if self.id >= 0 {
+                    self.clone()
+                } else {
+                    other.clone()
+                });
+            }
             return Err(Error::arrow(format!(
                 "Attempt to intersect blob and non-blob field: {}",
                 self.name
@@ -847,7 +886,7 @@ impl Field {
                 .iter()
                 .filter_map(|c| {
                     if let Some(other_child) = other.child(&c.name) {
-                        let intersection = c.intersection(other_child).ok()?;
+                        let intersection = c.do_intersection(other_child, ignore_types).ok()?;
                         Some(intersection)
                     } else {
                         None
@@ -1038,7 +1077,6 @@ impl Field {
 
     // Check if field has metadata `packed` set to true, this check is case insensitive.
     pub fn is_packed_struct(&self) -> bool {
-        const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
         PACKED_KEYS.iter().any(|key| {
             self.metadata
                 .get(*key)
@@ -1847,6 +1885,14 @@ mod tests {
     #[test]
     fn blob_unloaded_mut_selects_layout_from_metadata() {
         let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let mut binary_field: Field = ArrowField::new("blob", DataType::LargeBinary, true)
+            .with_metadata(metadata.clone())
+            .try_into()
+            .unwrap();
+        binary_field.binary_blob_mut();
+        assert!(binary_field.metadata.contains_key(BLOB_META_KEY));
+        assert!(!binary_field.is_blob_v2());
+
         let mut field: Field = ArrowField::new("blob", DataType::LargeBinary, true)
             .with_metadata(metadata)
             .try_into()
@@ -1854,6 +1900,12 @@ mod tests {
         field.unloaded_mut();
         assert_eq!(field.children.len(), 2);
         assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type);
+        assert!(field.is_blob());
+        assert!(!field.is_blob_v2());
+        field.unloaded_mut();
+        assert_eq!(field.children.len(), 2);
+        assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.is_blob_v2());
 
         let metadata =
             HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]);
@@ -1874,6 +1926,12 @@ mod tests {
         field.unloaded_mut();
         assert_eq!(field.children.len(), 5);
         assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY));
+        assert!(field.is_blob_v2());
+        field.unloaded_mut();
+        assert_eq!(field.children.len(), 5);
+        assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY));
     }
 
     #[test]
@@ -1944,4 +2002,43 @@ mod tests {
             .unwrap();
         assert_eq!(unloaded_projected, unloaded);
     }
+
+    #[test]
+    fn blob_descriptor_projection_preserves_synthetic_children() {
+        let metadata =
+            HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]);
+        let mut blob: Field = ArrowField::new(
+            "blob",
+            DataType::Struct(
+                vec![
+                    ArrowField::new("data", DataType::LargeBinary, true),
+                    ArrowField::new("uri", DataType::Utf8, true),
+                ]
+                .into(),
+            ),
+            true,
+        )
+        .with_metadata(metadata)
+        .try_into()
+        .unwrap();
+        let mut next_id = 0;
+        blob.set_id(-1, &mut next_id);
+
+        let schema = Arc::new(crate::datatypes::Schema {
+            fields: vec![blob],
+            metadata: HashMap::new(),
+        });
+        let descriptor_schema = Projection::full(schema)
+            .with_blob_handling(crate::datatypes::BlobHandling::BlobsDescriptions)
+            .to_bare_schema();
+        assert!(
+            descriptor_schema.fields[0]
+                .children
+                .iter()
+                .all(|child| child.id == -1)
+        );
+
+        let projected = Projection::full(Arc::new(descriptor_schema)).to_bare_schema();
+        assert_eq!(projected.fields[0].children.len(), 5);
+    }
 }
diff --git a/rust/lance-core/src/datatypes/schema.rs b/rust/lance-core/src/datatypes/schema.rs
index 7f2cbc02f07..6f9fc61b334 100644
--- a/rust/lance-core/src/datatypes/schema.rs
+++ b/rust/lance-core/src/datatypes/schema.rs
@@ -1071,6 +1071,17 @@ pub enum BlobHandling {
 }
 
 impl BlobHandling {
+    fn should_load_binary(&self, field: &Field) -> bool {
+        if !field.is_blob() {
+            return false;
+        }
+        match self {
+            Self::AllBinary => true,
+            Self::SomeBlobsBinary(set) | Self::SomeBinary(set) => set.contains(&(field.id as u32)),
+            Self::BlobsDescriptions | Self::AllDescriptions => false,
+        }
+    }
+
     fn should_unload(&self, field: &Field) -> bool {
         // Blob v2 columns are Structs, so we need to treat any blob-marked field as unloadable
         // even if the physical data type is not binary-like.
@@ -1086,10 +1097,34 @@ impl BlobHandling {
         }
     }
 
+    /// Whether `field` will be projected as a lightweight blob *description*
+    /// (offset + size) rather than its full binary value under this handling.
+    ///
+    /// A description is tiny and cheap to read eagerly; the full binary value is
+    /// not. Materialization heuristics use this to decide early vs late loading.
+    pub fn returns_description(&self, field: &Field) -> bool {
+        self.should_unload(field)
+    }
+
+    /// Apply this blob handling policy to a projected field tree.
+    ///
+    /// Blob descriptor modes convert blob leaves to descriptor views. Binary
+    /// modes convert selected blob leaves to `LargeBinary`. Non-blob nested
+    /// fields are preserved while their children are handled recursively.
     pub fn unload_if_needed(&self, mut field: Field) -> Field {
+        if self.should_load_binary(&field) {
+            field.binary_blob_mut();
+            return field;
+        }
         if self.should_unload(&field) {
             field.unloaded_mut();
+            return field;
         }
+        field.children = field
+            .children
+            .into_iter()
+            .map(|child| self.unload_if_needed(child))
+            .collect();
         field
     }
 }
diff --git a/rust/lance-core/src/error.rs b/rust/lance-core/src/error.rs
index 2a6340da492..8c0684d64f4 100644
--- a/rust/lance-core/src/error.rs
+++ b/rust/lance-core/src/error.rs
@@ -8,6 +8,52 @@ use snafu::{IntoError as _, Location, Snafu};
 
 type BoxedError = Box<dyn std::error::Error + Send + Sync + 'static>;
 
+#[cfg(feature = "backtrace")]
+mod backtrace_support {
+    use std::backtrace::Backtrace;
+
+    use snafu::{AsBacktrace, GenerateImplicitData};
+
+    #[derive(Debug)]
+    pub struct MaybeBacktrace(pub Option<Backtrace>);
+
+    impl GenerateImplicitData for MaybeBacktrace {
+        fn generate() -> Self {
+            Self(<Option<Backtrace>>::generate())
+        }
+    }
+
+    impl AsBacktrace for MaybeBacktrace {
+        fn as_backtrace(&self) -> Option<&Backtrace> {
+            self.0.as_ref()
+        }
+    }
+}
+
+#[cfg(not(feature = "backtrace"))]
+mod backtrace_support {
+    use std::backtrace::Backtrace;
+
+    use snafu::{AsBacktrace, GenerateImplicitData};
+
+    #[derive(Debug)]
+    pub struct MaybeBacktrace;
+
+    impl GenerateImplicitData for MaybeBacktrace {
+        fn generate() -> Self {
+            Self
+        }
+    }
+
+    impl AsBacktrace for MaybeBacktrace {
+        fn as_backtrace(&self) -> Option<&Backtrace> {
+            None
+        }
+    }
+}
+
+use backtrace_support::MaybeBacktrace;
+
 /// Error for when a requested field is not found in a schema.
 ///
 /// This error computes suggestions lazily (only when displayed) to avoid
@@ -81,18 +127,24 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Dataset already exists: {uri}, {location}"))]
     DatasetAlreadyExists {
         uri: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Append with different schema: {difference}, location: {location}"))]
     SchemaMismatch {
         difference: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Dataset at path {path} was not found: {source}, {location}"))]
     DatasetNotFound {
@@ -100,6 +152,8 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Encountered corrupt file {path}: {source}, {location}"))]
     CorruptFile {
@@ -107,13 +161,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
-        // TODO: add backtrace?
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Not supported: {source}, {location}"))]
     NotSupported {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Commit conflict for version {version}: {source}, {location}"))]
     CommitConflict {
@@ -121,12 +178,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Incompatible transaction: {source}, {location}"))]
     IncompatibleTransaction {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Retryable commit conflict for version {version}: {source}, {location}"))]
     RetryableCommitConflict {
@@ -134,12 +195,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Too many concurrent writers. {message}, {location}"))]
     TooMuchWriteContention {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Operation timed out: {message}, {location}"))]
     Timeout {
@@ -154,54 +219,72 @@ pub enum Error {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("A prerequisite task failed: {message}, {location}"))]
     PrerequisiteFailed {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Unprocessable: {message}, {location}"))]
     Unprocessable {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Arrow): {message}, {location}"))]
     Arrow {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Schema): {message}, {location}"))]
     Schema {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Not found: {uri}, {location}"))]
     NotFound {
         uri: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(IO): {source}, {location}"))]
     IO {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Index): {message}, {location}"))]
     Index {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Lance index not found: {identity}, {location}"))]
     IndexNotFound {
         identity: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Cannot infer storage location from: {message}"))]
     InvalidTableLocation { message: String },
@@ -209,21 +292,28 @@ pub enum Error {
     Stop,
     #[snafu(display("Wrapped error: {error}, {location}"))]
     Wrapped {
+        #[snafu(source)]
         error: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Cloned error: {message}, {location}"))]
     Cloned {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Query Execution error: {message}, {location}"))]
     Execution {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Ref is invalid: {message}"))]
     InvalidRef { message: String },
@@ -242,12 +332,16 @@ pub enum Error {
         minor_version: u16,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Namespace error: {source}, {location}"))]
     Namespace {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     /// External error passed through from user code.
     ///
@@ -283,6 +377,65 @@ pub enum Error {
 }
 
 impl Error {
+    /// Returns the captured Rust backtrace, if available.
+    ///
+    /// Requires the `backtrace` feature to be enabled at compile time
+    /// and `RUST_BACKTRACE=1` at runtime.
+    #[cfg(feature = "backtrace")]
+    pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> {
+        match self {
+            Self::InvalidInput { backtrace, .. }
+            | Self::DatasetAlreadyExists { backtrace, .. }
+            | Self::SchemaMismatch { backtrace, .. }
+            | Self::DatasetNotFound { backtrace, .. }
+            | Self::CorruptFile { backtrace, .. }
+            | Self::NotSupported { backtrace, .. }
+            | Self::CommitConflict { backtrace, .. }
+            | Self::IncompatibleTransaction { backtrace, .. }
+            | Self::RetryableCommitConflict { backtrace, .. }
+            | Self::TooMuchWriteContention { backtrace, .. }
+            | Self::Internal { backtrace, .. }
+            | Self::PrerequisiteFailed { backtrace, .. }
+            | Self::Unprocessable { backtrace, .. }
+            | Self::Arrow { backtrace, .. }
+            | Self::Schema { backtrace, .. }
+            | Self::NotFound { backtrace, .. }
+            | Self::IO { backtrace, .. }
+            | Self::Index { backtrace, .. }
+            | Self::IndexNotFound { backtrace, .. }
+            | Self::Wrapped { backtrace, .. }
+            | Self::Cloned { backtrace, .. }
+            | Self::Execution { backtrace, .. }
+            | Self::VersionConflict { backtrace, .. }
+            | Self::Namespace { backtrace, .. } => {
+                use snafu::AsBacktrace;
+                backtrace.as_backtrace()
+            }
+            // Variants without a backtrace field — listed explicitly so that
+            // adding a new variant with a backtrace field triggers a compiler error.
+            Self::InvalidTableLocation { .. }
+            | Self::Stop
+            | Self::InvalidRef { .. }
+            | Self::RefConflict { .. }
+            | Self::RefNotFound { .. }
+            | Self::Cleanup { .. }
+            | Self::VersionNotFound { .. }
+            | Self::External { .. }
+            | Self::FieldNotFound { .. }
+            | Self::Timeout { .. }
+            | Self::DiskCapExceeded { .. }
+            | Self::Fenced { .. } => None,
+        }
+    }
+
+    /// Returns the captured Rust backtrace, if available.
+    ///
+    /// Always returns `None` when the `backtrace` feature is not enabled.
+    #[cfg(not(feature = "backtrace"))]
+    pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> {
+        None
+    }
+
     #[track_caller]
     pub fn corrupt_file(path: object_store::path::Path, message: impl Into<String>) -> Self {
         CorruptFileSnafu { path }.into_error(message.into().into())
@@ -367,9 +520,20 @@ impl Error {
         NotFoundSnafu { uri: uri.into() }.build()
     }
 
+    /// Return whether this error or one of its typed sources is a missing object.
+    pub fn is_not_found(&self) -> bool {
+        match self {
+            Self::NotFound { .. } => true,
+            Self::IO { source, .. } | Self::Wrapped { error: source, .. } => {
+                error_source_is_not_found(source.as_ref())
+            }
+            _ => false,
+        }
+    }
+
     #[track_caller]
     pub fn wrapped(error: BoxedError) -> Self {
-        WrappedSnafu { error }.build()
+        WrappedSnafu.into_error(error)
     }
 
     #[track_caller]
@@ -548,6 +712,17 @@ impl Error {
     }
 }
 
+fn error_source_is_not_found(source: &(dyn std::error::Error + 'static)) -> bool {
+    if let Some(error) = source.downcast_ref::<Error>() {
+        return error.is_not_found();
+    }
+    if let Some(error) = source.downcast_ref::<object_store::Error>() {
+        return matches!(error, object_store::Error::NotFound { .. })
+            || std::error::Error::source(error).is_some_and(error_source_is_not_found);
+    }
+    source.source().is_some_and(error_source_is_not_found)
+}
+
 pub trait LanceOptionExt<T> {
     /// Unwraps an option, returning an internal error if the option is None.
     ///
@@ -611,7 +786,11 @@ impl From<std::io::Error> for Error {
 impl From<object_store::Error> for Error {
     #[track_caller]
     fn from(e: object_store::Error) -> Self {
-        Self::io_source(box_error(e))
+        match e {
+            // source intentionally dropped; Error::NotFound carries only the path
+            object_store::Error::NotFound { path, .. } => Self::not_found(path),
+            other => Self::io_source(box_error(other)),
+        }
     }
 }
 
@@ -717,6 +896,17 @@ impl From<datafusion_common::DataFusionError> for Error {
                 Self::not_supported_source(box_error(e))
             }
             datafusion_common::DataFusionError::Execution(..) => Self::execution(e.to_string()),
+            datafusion_common::DataFusionError::Shared(shared) => {
+                // DataFusion shares an error across consumers (e.g. a join's
+                // build-side error fanned out to every probe partition) behind an
+                // `Arc`. If we are the sole owner we can recurse for full fidelity;
+                // otherwise the inner error can't be moved out, so we preserve its
+                // message under the execution category (its concrete type is lost).
+                match std::sync::Arc::try_unwrap(shared) {
+                    Ok(inner) => Self::from(inner),
+                    Err(shared) => Self::execution(shared.to_string()),
+                }
+            }
             datafusion_common::DataFusionError::External(source) => {
                 // Try to downcast to lance_core::Error first
                 match source.downcast::<Self>() {
@@ -749,14 +939,46 @@ pub fn get_caller_location() -> &'static std::panic::Location<'static> {
 /// Wrap an error in a new error type that implements Clone
 ///
 /// This is useful when two threads/streams share a common fallible source
-/// The base error will always have the full error.  Any cloned results will
-/// only have Error::Cloned with the to_string of the base error.
+/// Definite not-found errors preserve typed source-chain detection and their
+/// human-readable representation. Timeout and I/O errors preserve their error
+/// categories. Other cloned results use Error::Cloned with the string
+/// representation of the base error.
 pub struct CloneableError(pub Error);
 
+struct DisplayError(Error);
+
+impl fmt::Debug for DisplayError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+impl fmt::Display for DisplayError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}
+
+impl std::error::Error for DisplayError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(&self.0)
+    }
+}
+
 impl Clone for CloneableError {
     #[track_caller]
     fn clone(&self) -> Self {
-        Self(Error::cloned(self.0.to_string()))
+        match &self.0 {
+            Error::NotFound { uri, .. } => Self(Error::wrapped(Box::new(DisplayError(
+                Error::not_found(uri.clone()),
+            )))),
+            error if error.is_not_found() => Self(Error::wrapped(Box::new(DisplayError(
+                Error::not_found(error.to_string()),
+            )))),
+            Error::Timeout { message, .. } => Self(Error::timeout(message.clone())),
+            Error::IO { source, .. } => Self(Error::io(source.to_string())),
+            error => Self(Error::cloned(error.to_string())),
+        }
     }
 }
 
@@ -772,8 +994,56 @@ impl<T: Clone> From<Result<T>> for CloneableResult<T> {
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::error::Error as _;
     use std::fmt;
 
+    #[test]
+    fn cloneable_error_preserves_not_found_contract() {
+        let original = CloneableError(Error::not_found("metadata.lance"));
+        let cloned = original.clone();
+        let cloned_again = cloned.clone();
+        assert!(matches!(original.0, Error::NotFound { .. }));
+        assert!(cloned.0.is_not_found());
+        assert!(cloned_again.0.is_not_found());
+        assert!(cloned.0.to_string().to_lowercase().contains("not found"));
+        assert!(
+            cloned_again
+                .0
+                .to_string()
+                .to_lowercase()
+                .contains("not found")
+        );
+        assert!(
+            format!("{:?}", cloned.0)
+                .to_lowercase()
+                .contains("not found")
+        );
+        assert!(cloned.0.source().is_some_and(|source| source.is::<Error>()
+            || source.source().is_some_and(|source| source.is::<Error>())));
+        let downstream_error = Error::wrapped(Box::new(Error::io_source(Box::new(
+            object_store::Error::Generic {
+                store: "N/A",
+                source: Box::new(cloned.0),
+            },
+        ))));
+        assert!(downstream_error.is_not_found());
+        assert!(
+            format!("{downstream_error:?}")
+                .to_lowercase()
+                .contains("not found")
+        );
+
+        let original = CloneableError(Error::timeout("metadata read timed out"));
+        let cloned = original.clone();
+        assert!(matches!(original.0, Error::Timeout { .. }));
+        assert!(matches!(cloned.0, Error::Timeout { .. }));
+
+        let original = CloneableError(Error::io("metadata read was denied"));
+        let cloned = original.clone();
+        assert!(matches!(original.0, Error::IO { .. }));
+        assert!(matches!(cloned.0, Error::IO { .. }));
+    }
+
     #[test]
     fn test_caller_location_capture() {
         let current_fn = get_caller_location();
@@ -796,6 +1066,41 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_caller_location_capture_not_found() {
+        let current_fn = get_caller_location();
+        let f: Box<dyn Fn() -> Result<()>> = Box::new(|| {
+            Err(object_store::Error::NotFound {
+                path: "some/path".to_string(),
+                source: "not found".into(),
+            })?;
+            Ok(())
+        });
+        match f().unwrap_err() {
+            Error::NotFound { location, .. } => {
+                // +2 is the beginning of object_store::Error::NotFound...
+                assert_eq!(location.line(), current_fn.line() + 2, "{}", location)
+            }
+            #[allow(unreachable_patterns)]
+            other => panic!("expected NotFound, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_object_store_not_found_converts_to_not_found() {
+        let os_err = object_store::Error::NotFound {
+            path: "test/path".to_string(),
+            source: "no such file".into(),
+        };
+        let lance_err: Error = os_err.into();
+        match lance_err {
+            Error::NotFound { uri, .. } => {
+                assert_eq!(uri, "test/path");
+            }
+            other => panic!("Expected NotFound, got {:?}", other),
+        }
+    }
+
     #[derive(Debug)]
     struct MyCustomError {
         code: i32,
@@ -1048,4 +1353,68 @@ mod test {
             _ => panic!("Expected InvalidInput variant, got {:?}", recovered),
         }
     }
+
+    #[test]
+    fn test_backtrace_accessor() {
+        // Verify that backtrace() returns the expected result based on feature state
+        let err = Error::io("test backtrace");
+        let bt = err.backtrace();
+        #[cfg(feature = "backtrace")]
+        {
+            // With the backtrace feature enabled, whether a backtrace is captured
+            // depends on the RUST_BACKTRACE env var at runtime. We just verify
+            // the accessor doesn't panic and returns a valid Option.
+            let _ = bt;
+        }
+        #[cfg(not(feature = "backtrace"))]
+        {
+            // Without the backtrace feature, this must always be None.
+            assert!(bt.is_none());
+        }
+    }
+
+    #[test]
+    fn test_backtrace_captured_when_feature_enabled() {
+        // Test that backtrace is actually captured when the feature is on and
+        // RUST_BACKTRACE=1 is set in the environment before the process starts.
+        //
+        // NOTE: std::backtrace::Backtrace caches the RUST_BACKTRACE env check,
+        // so set_var at runtime does not reliably enable capture. This test
+        // verifies the accessor works correctly in both cases:
+        // - If RUST_BACKTRACE=1 was set before the test binary started, we get Some.
+        // - If not, we get None (even with the feature on), which is expected.
+        #[cfg(feature = "backtrace")]
+        {
+            let err = Error::io("backtrace capture test");
+            if std::env::var("RUST_BACKTRACE").is_ok() {
+                assert!(
+                    err.backtrace().is_some(),
+                    "Expected a backtrace when RUST_BACKTRACE=1 and backtrace feature is enabled"
+                );
+            }
+            // When RUST_BACKTRACE is not set, backtrace() may return None even
+            // with the feature enabled — this is correct runtime gating behavior.
+        }
+        #[cfg(not(feature = "backtrace"))]
+        {
+            let err = Error::io("backtrace capture test");
+            assert!(err.backtrace().is_none());
+        }
+    }
+
+    #[test]
+    fn test_backtrace_returns_none_for_variants_without_location() {
+        let err = Error::InvalidTableLocation {
+            message: "test".to_string(),
+        };
+        assert!(err.backtrace().is_none());
+
+        let err = Error::InvalidRef {
+            message: "test".to_string(),
+        };
+        assert!(err.backtrace().is_none());
+
+        let err = Error::Stop;
+        assert!(err.backtrace().is_none());
+    }
 }
diff --git a/rust/lance-core/src/utils/cpu.rs b/rust/lance-core/src/utils/cpu.rs
index 4e7ab01871d..c4d5a976cbb 100644
--- a/rust/lance-core/src/utils/cpu.rs
+++ b/rust/lance-core/src/utils/cpu.rs
@@ -1,14 +1,29 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::fmt;
 use std::sync::LazyLock;
 
-/// A level of SIMD support for some feature
+/// A level of SIMD support for some feature.
+///
+/// `#[non_exhaustive]` so future tiers (e.g. AVX-512 BF16, AMX) can be added
+/// without breaking external `match` consumers.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
 pub enum SimdSupport {
     None,
     Neon,
     Sse,
+    /// AVX (256-bit float ops) but no FMA and no AVX2.
+    /// Intel Sandy Bridge / Ivy Bridge.
+    Avx,
+    /// AVX + FMA but no AVX2.
+    /// AMD Piledriver / Steamroller / FX-7500.
+    AvxFma,
+    /// AVX2 + FMA. Intel Haswell / AMD Excavator and later.
+    ///
+    /// Selecting this tier asserts FMA is present: the kernels it dispatches to
+    /// are `#[target_feature(enable = "avx,fma")]`.
     Avx2,
     Avx512,
     Avx512FP16,
@@ -16,6 +31,132 @@ pub enum SimdSupport {
     Lasx,
 }
 
+impl fmt::Display for SimdSupport {
+    /// Formats the tier name in lowercase, matching pyarrow's
+    /// `runtime_info().simd_level` convention.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let name = match self {
+            Self::None => "none",
+            Self::Neon => "neon",
+            Self::Sse => "sse",
+            Self::Avx => "avx",
+            Self::AvxFma => "avx_fma",
+            Self::Avx2 => "avx2",
+            Self::Avx512 => "avx512",
+            Self::Avx512FP16 => "avx512_fp16",
+            Self::Lsx => "lsx",
+            Self::Lasx => "lasx",
+        };
+        f.write_str(name)
+    }
+}
+
+/// Snapshot of the SIMD tier lance dispatches to on the current host, plus the
+/// raw CPU features detected for diagnostic purposes.
+///
+/// Mirrors the role of `pyarrow.runtime_info()`: a single, cheap call users can
+/// make to verify which SIMD tier the runtime selected and what underlying
+/// features the host advertises. Obtain one with [`simd_info()`].
+#[derive(Debug, Clone)]
+pub struct SimdInfo {
+    /// The SIMD tier lance dispatches to at runtime on this host.
+    pub tier: SimdSupport,
+    /// The architecture name (e.g. "x86_64", "aarch64", "loongarch64").
+    pub target_arch: &'static str,
+    /// Raw CPU feature flags detected on this host (x86_64 only; empty on
+    /// other architectures). Each entry is a feature name like "avx2",
+    /// "fma", "avx512f", "popcnt", etc.
+    pub host_features: Vec<&'static str>,
+}
+
+/// Returns a snapshot of the SIMD tier lance is using on this host along with
+/// the raw CPU feature flags that drove the decision.
+///
+/// Useful for performance debugging and giving users a way to verify which
+/// dispatch tier they are hitting without rebuilding lance. See [`SimdInfo`]
+/// for the meaning of each field and [`SimdSupport`] for the tier values.
+///
+/// # Examples
+///
+/// ```
+/// use lance_core::utils::cpu::simd_info;
+///
+/// let info = simd_info();
+/// println!("dispatching to {} on {}", info.tier, info.target_arch);
+/// ```
+pub fn simd_info() -> SimdInfo {
+    SimdInfo {
+        tier: *SIMD_SUPPORT,
+        target_arch: std::env::consts::ARCH,
+        host_features: detect_host_features(),
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn detect_host_features() -> Vec<&'static str> {
+    // Each call must be inline: `is_x86_feature_detected!` does its own custom
+    // input parsing and rejects feature names received via a `macro_rules!`
+    // `:literal` metavariable on some toolchains.
+    let mut features = Vec::with_capacity(17);
+    if is_x86_feature_detected!("sse2") {
+        features.push("sse2");
+    }
+    if is_x86_feature_detected!("sse3") {
+        features.push("sse3");
+    }
+    if is_x86_feature_detected!("ssse3") {
+        features.push("ssse3");
+    }
+    if is_x86_feature_detected!("sse4.1") {
+        features.push("sse4.1");
+    }
+    if is_x86_feature_detected!("sse4.2") {
+        features.push("sse4.2");
+    }
+    if is_x86_feature_detected!("popcnt") {
+        features.push("popcnt");
+    }
+    if is_x86_feature_detected!("avx") {
+        features.push("avx");
+    }
+    if is_x86_feature_detected!("avx2") {
+        features.push("avx2");
+    }
+    if is_x86_feature_detected!("fma") {
+        features.push("fma");
+    }
+    if is_x86_feature_detected!("f16c") {
+        features.push("f16c");
+    }
+    if is_x86_feature_detected!("bmi1") {
+        features.push("bmi1");
+    }
+    if is_x86_feature_detected!("bmi2") {
+        features.push("bmi2");
+    }
+    if is_x86_feature_detected!("avx512f") {
+        features.push("avx512f");
+    }
+    if is_x86_feature_detected!("avx512bw") {
+        features.push("avx512bw");
+    }
+    if is_x86_feature_detected!("avx512cd") {
+        features.push("avx512cd");
+    }
+    if is_x86_feature_detected!("avx512dq") {
+        features.push("avx512dq");
+    }
+    if is_x86_feature_detected!("avx512vl") {
+        features.push("avx512vl");
+    }
+    features
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+fn detect_host_features() -> Vec<&'static str> {
+    Vec::new()
+}
+
 /// Support for SIMD operations
 pub static SIMD_SUPPORT: LazyLock<SimdSupport> = LazyLock::new(|| {
     #[cfg(all(target_arch = "aarch64", any(target_os = "ios", target_os = "tvos")))]
@@ -42,8 +183,19 @@ pub static SIMD_SUPPORT: LazyLock<SimdSupport> = LazyLock::new(|| {
             } else {
                 SimdSupport::Avx512
             }
-        } else if is_x86_feature_detected!("avx2") {
+        } else if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
+            // FMA is checked explicitly: every kernel selected for this tier is
+            // `#[target_feature(enable = "avx,fma")]`, and AVX2 does not imply
+            // FMA in the ISA. Every shipping AVX2 part has FMA, so this only
+            // guards against a host that would otherwise take an FMA kernel
+            // without FMA.
             SimdSupport::Avx2
+        } else if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") {
+            // AMD Piledriver / Steamroller / FX-7500: 256-bit float ops + FMA but no AVX2.
+            SimdSupport::AvxFma
+        } else if is_x86_feature_detected!("avx") {
+            // Intel Sandy Bridge / Ivy Bridge: 256-bit float ops without FMA.
+            SimdSupport::Avx
         } else {
             SimdSupport::None
         }
@@ -138,3 +290,68 @@ mod aarch64 {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[test]
+    fn simd_info_exposes_tier() {
+        let info = simd_info();
+        assert_eq!(info.target_arch, std::env::consts::ARCH);
+        // Tier should match the detected SIMD support.
+        assert_eq!(info.tier, *SIMD_SUPPORT);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn simd_info_features_include_baseline() {
+        let info = simd_info();
+        // The x86_64 ABI mandates SSE2, so it must always be present on this
+        // architecture.
+        assert!(info.host_features.contains(&"sse2"));
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[test]
+    fn simd_info_features_empty_off_x86_64() {
+        let info = simd_info();
+        assert!(info.host_features.is_empty());
+    }
+
+    /// The `Avx2` and `AvxFma` tiers both dispatch to kernels declared
+    /// `#[target_feature(enable = "avx,fma")]`, so neither may be selected on a
+    /// host without FMA. AVX2 does not imply FMA in the ISA, so the detection
+    /// checks it explicitly. (`Avx512*` is excluded: its kernels declare
+    /// `avx512f`, which is what `has_avx512` verifies.)
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn avx_fma_tiers_are_only_selected_when_fma_is_detected() {
+        if matches!(*SIMD_SUPPORT, SimdSupport::Avx2 | SimdSupport::AvxFma) {
+            assert!(
+                is_x86_feature_detected!("fma"),
+                "tier {} dispatches to avx,fma kernels but the host has no FMA",
+                *SIMD_SUPPORT
+            );
+        }
+    }
+
+    #[rstest]
+    #[case::none(SimdSupport::None, "none")]
+    #[case::neon(SimdSupport::Neon, "neon")]
+    #[case::sse(SimdSupport::Sse, "sse")]
+    #[case::avx(SimdSupport::Avx, "avx")]
+    #[case::avx_fma(SimdSupport::AvxFma, "avx_fma")]
+    #[case::avx2(SimdSupport::Avx2, "avx2")]
+    #[case::avx512(SimdSupport::Avx512, "avx512")]
+    #[case::avx512_fp16(SimdSupport::Avx512FP16, "avx512_fp16")]
+    #[case::lsx(SimdSupport::Lsx, "lsx")]
+    #[case::lasx(SimdSupport::Lasx, "lasx")]
+    fn simd_support_display_matches_lowercase_convention(
+        #[case] tier: SimdSupport,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(tier.to_string(), expected);
+    }
+}
diff --git a/rust/lance-core/src/utils/tokio.rs b/rust/lance-core/src/utils/tokio.rs
index 46c9475665b..fd33fc6b216 100644
--- a/rust/lance-core/src/utils/tokio.rs
+++ b/rust/lance-core/src/utils/tokio.rs
@@ -66,11 +66,15 @@ static RUNTIME_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false);
 
 static ATFORK_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false);
 
-fn global_cpu_runtime() -> &'static mut Runtime {
+fn global_cpu_runtime() -> &'static Runtime {
     loop {
         let ptr = CPU_RUNTIME.load(Ordering::SeqCst);
         if !ptr.is_null() {
-            return unsafe { &mut *ptr };
+            // SAFETY: `ptr` was produced by `Box::into_raw` below and is only ever
+            // reset to null by `atfork_tokio_child` in the forked child (single-
+            // threaded, async-signal context). The `Box` is never reclaimed, so the
+            // `Runtime` lives for the rest of the process.
+            return unsafe { &*ptr };
         }
         if !RUNTIME_INSTALLED.fetch_or(true, Ordering::SeqCst) {
             break;
@@ -82,7 +86,9 @@ fn global_cpu_runtime() -> &'static mut Runtime {
     }
     let new_ptr = Box::into_raw(Box::new(create_runtime()));
     CPU_RUNTIME.store(new_ptr, Ordering::SeqCst);
-    unsafe { &mut *new_ptr }
+    // SAFETY: `new_ptr` was just obtained from `Box::into_raw`, so it is non-null,
+    // aligned, and points to a live `Runtime` that is never reclaimed.
+    unsafe { &*new_ptr }
 }
 
 /// After a fork() operation, force re-creation of the BackgroundExecutor. Note: this function
@@ -108,6 +114,43 @@ fn install_atfork() {}
 ///
 /// This can also be used to convert a big chunk of synchronous work into a future
 /// so that it can be run in parallel with something like StreamExt::buffered()
+///
+/// # Only hand over substantial CPU work
+///
+/// Dispatching to the pool has real overhead (a `spawn_blocking` hop plus a oneshot
+/// channel round trip). As a rule of thumb the closure should be expected to do at
+/// least ~100µs of CPU work; below that the thread-pool overhead is likely to
+/// outweigh any parallelism benefit, and the work is better left inline.
+///
+/// # The task must never wait on anything
+///
+/// The CPU pool is sized to [`get_num_compute_intensive_cpus`], which is
+/// `max(1, num_cpus - LANCE_IO_CORE_RESERVATION)`. On a big host that is plenty of
+/// workers (e.g. 62 on a 64-core box), but in resource-constrained environments it can
+/// collapse to a **single blocking thread** — on machines with `<= 3` visible CPUs
+/// (1-vCPU VMs, CI runners, CPU-limited Kubernetes pods) the pool has exactly one
+/// worker. A closure passed to `spawn_cpu` occupies one of these threads for its entire
+/// lifetime, including any time it spends *parked*. So the closure must only consume
+/// CPU and return; it must
+/// **never** block, wait, or park. Concretely, the closure must not, directly or
+/// transitively:
+///
+/// * **No channels** — no blocking send/recv (`send_blocking`, blocking `recv`, etc.).
+///   A full/empty channel parks the thread, and whatever would drain/fill the channel
+///   may need the same pool to run.
+/// * **No I/O** — no file, network, or object-store reads/writes, and no disk spills.
+///   I/O parks the thread while making no progress on CPU work.
+/// * **No locks** — no acquiring a contended lock (or any lock that is held across an
+///   `.await` elsewhere). Waiting for the lock parks the thread.
+/// * **No `block_on` / `.blocking_*`** — never drive or wait on another async task
+///   from inside the closure.
+///
+/// If any of these hold, the parked thread can starve the exact work that would
+/// unblock it, deadlocking the whole pool with no timeout and no error — a silent
+/// hang at 0% CPU. (See <https://github.com/lancedb/lance/pull/7423>.) When work
+/// needs to wait on a channel/lock/I/O, keep the waiting in an async task and only
+/// hand the pure-CPU portion to `spawn_cpu`, e.g. build each batch with `spawn_cpu`
+/// and dispatch it with `tx.send(batch).await` in the surrounding async code.
 pub fn spawn_cpu<
     E: std::error::Error + Send + 'static,
     F: FnOnce() -> std::result::Result<R, E> + Send + 'static,
diff --git a/rust/lance-datafusion/src/exec.rs b/rust/lance-datafusion/src/exec.rs
index 8f346f45612..c9822250651 100644
--- a/rust/lance-datafusion/src/exec.rs
+++ b/rust/lance-datafusion/src/exec.rs
@@ -6,6 +6,7 @@
 use std::{
     collections::HashMap,
     fmt::{self, Formatter},
+    num::NonZero,
     sync::{Arc, Mutex, OnceLock},
     time::Duration,
 };
@@ -14,9 +15,8 @@ use chrono::{DateTime, Utc};
 
 use arrow_array::RecordBatch;
 use arrow_schema::Schema as ArrowSchema;
-use datafusion::physical_plan::metrics::MetricType;
 use datafusion::{
-    catalog::streaming::StreamingTable,
+    catalog::{TableProvider, streaming::StreamingTable},
     dataframe::DataFrame,
     execution::{
         TaskContext,
@@ -26,16 +26,19 @@ use datafusion::{
         runtime_env::RuntimeEnvBuilder,
     },
     physical_plan::{
-        DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream,
+        DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+        SendableRecordBatchStream,
         analyze::AnalyzeExec,
         coalesce_partitions::CoalescePartitionsExec,
         display::DisplayableExecutionPlan,
         execution_plan::{Boundedness, CardinalityEffect, EmissionType},
         metrics::MetricValue,
+        sorts::sort_preserving_merge::SortPreservingMergeExec,
         stream::RecordBatchStreamAdapter,
         streaming::PartitionStream,
     },
 };
+use datafusion::{execution::memory_pool::TrackConsumersPool, physical_plan::metrics::MetricType};
 use datafusion_common::{DataFusionError, Statistics};
 use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
 
@@ -152,10 +155,6 @@ impl ExecutionPlan for OneShotExec {
         "OneShotExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.schema.clone()
     }
@@ -243,10 +242,6 @@ impl ExecutionPlan for TracedExec {
         "TracedExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -310,7 +305,7 @@ impl std::fmt::Debug for LanceExecutionOptions {
     }
 }
 
-const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 100 * 1024 * 1024;
+const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 150 * 1024 * 1024;
 const DEFAULT_LANCE_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB
 
 impl LanceExecutionOptions {
@@ -366,12 +361,21 @@ pub fn new_session_context(options: &LanceExecutionOptions) -> SessionContext {
         session_config = session_config.with_target_partitions(target_partition);
     }
     if options.use_spilling() {
+        // The default 10MB sort spill reservation seems to be too small for many common cases.
+        //
+        // There currently is no reasonable guidance provided by DataFusion for setting this value.
+        // We bump this to 40MB but try a smaller value if the mem pool is small.
+        let sort_spill_reservation_bytes =
+            (options.mem_pool_size() / 3).min(40 * 1024 * 1024) as usize;
+        session_config =
+            session_config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes);
         let disk_manager_builder = DiskManagerBuilder::default()
             .with_max_temp_directory_size(options.max_temp_directory_size());
         runtime_env_builder = runtime_env_builder
             .with_disk_manager_builder(disk_manager_builder)
-            .with_memory_pool(Arc::new(FairSpillPool::new(
-                options.mem_pool_size() as usize
+            .with_memory_pool(Arc::new(TrackConsumersPool::new(
+                FairSpillPool::new(options.mem_pool_size() as usize),
+                NonZero::try_from(16).unwrap(),
             )));
     }
     let runtime_env = runtime_env_builder.build_arc().unwrap();
@@ -610,8 +614,17 @@ pub fn execute_plan(
     // Coalesce to a single partition if the optimizer left more than one.
     // EnforceDistribution may remove RepartitionExec(1) nodes when the parent
     // declares UnspecifiedDistribution, leaving multi-partition plans here.
+    //
+    // If the plan carries an output ordering (e.g. a top-k `SortExec` whose
+    // result was later repartitioned to parallelize downstream operators),
+    // a plain `CoalescePartitionsExec` would scramble that order because it
+    // merges partitions in scheduling-dependent order. Use an order-preserving
+    // merge in that case instead, mirroring what `EnforceDistribution` itself
+    // does when it needs to merge an ordered, multi-partition plan.
     let plan: Arc<dyn ExecutionPlan> = if plan.properties().partitioning.partition_count() == 1 {
         plan
+    } else if let Some(ordering) = plan.output_ordering() {
+        Arc::new(SortPreservingMergeExec::new(ordering.clone(), plan))
     } else {
         Arc::new(CoalescePartitionsExec::new(plan))
     };
@@ -640,7 +653,8 @@ pub async fn analyze_plan(
     let analyze = Arc::new(AnalyzeExec::new(
         true,
         true,
-        vec![MetricType::SUMMARY],
+        vec![MetricType::Summary],
+        None,
         plan,
         schema,
     ));
@@ -867,6 +881,49 @@ impl SessionContextExt for SessionContext {
     }
 }
 
+/// Scan a [`TableProvider`] into a single-partition [`SendableRecordBatchStream`].
+///
+/// Multi-partition providers are coalesced into a single partition. This adapts a
+/// re-scannable provider back into the one stream the writer pipeline consumes;
+/// re-scanning the same provider (e.g. on a write retry) yields a fresh stream.
+///
+/// # Examples
+///
+/// ```
+/// # use std::sync::Arc;
+/// # use arrow_array::{Int32Array, RecordBatch};
+/// # use arrow_schema::{DataType, Field, Schema};
+/// # use datafusion::catalog::TableProvider;
+/// # use datafusion::datasource::MemTable;
+/// # use futures::TryStreamExt;
+/// # use lance_datafusion::exec::provider_to_stream;
+/// # #[tokio::main]
+/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+/// let batch =
+///     RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?;
+/// let provider: Arc<dyn TableProvider> = Arc::new(MemTable::try_new(schema, vec![vec![batch]])?);
+///
+/// // A re-scannable provider yields a fresh stream on each call.
+/// let batches: Vec<RecordBatch> = provider_to_stream(provider).await?.try_collect().await?;
+/// assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
+/// # Ok(())
+/// # }
+/// ```
+pub async fn provider_to_stream(
+    provider: Arc<dyn TableProvider>,
+) -> Result<SendableRecordBatchStream> {
+    let ctx = SessionContext::new();
+    let plan = provider.scan(&ctx.state(), None, &[], None).await?;
+    let plan: Arc<dyn ExecutionPlan> =
+        if plan.properties().output_partitioning().partition_count() > 1 {
+            Arc::new(CoalescePartitionsExec::new(plan))
+        } else {
+            plan
+        };
+    Ok(plan.execute(0, ctx.task_ctx())?)
+}
+
 #[derive(Clone, Debug)]
 pub struct StrictBatchSizeExec {
     input: Arc<dyn ExecutionPlan>,
@@ -894,10 +951,6 @@ impl ExecutionPlan for StrictBatchSizeExec {
         "StrictBatchSizeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
@@ -938,7 +991,7 @@ impl ExecutionPlan for StrictBatchSizeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion_common::Result<Statistics> {
+    ) -> datafusion_common::Result<std::sync::Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
@@ -1000,10 +1053,6 @@ impl ExecutionPlan for HardCapBatchSizeExec {
         "HardCapBatchSizeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
@@ -1065,7 +1114,7 @@ impl ExecutionPlan for HardCapBatchSizeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion_common::Result<Statistics> {
+    ) -> datafusion_common::Result<std::sync::Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
diff --git a/rust/lance-datafusion/src/logical_expr.rs b/rust/lance-datafusion/src/logical_expr.rs
index 0eed438dae7..db9abd7e204 100644
--- a/rust/lance-datafusion/src/logical_expr.rs
+++ b/rust/lance-datafusion/src/logical_expr.rs
@@ -51,14 +51,10 @@ pub fn resolve_column_type(expr: &Expr, schema: &Schema) -> Option<DataType> {
                 field_path.push(c.name.as_str());
                 break;
             }
-            Expr::ScalarFunction(udf) => {
-                if udf.name() == GetFieldFunc::default().name() {
-                    let name = get_as_string_scalar_opt(&udf.args[1])?;
-                    field_path.push(name);
-                    current_expr = &udf.args[0];
-                } else {
-                    return None;
-                }
+            Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => {
+                let name = get_as_string_scalar_opt(&udf.args[1])?;
+                field_path.push(name);
+                current_expr = &udf.args[0];
             }
             _ => return None,
         }
@@ -285,8 +281,10 @@ pub fn field_path_to_expr(field_path: &str) -> Result<Expr> {
         )));
     }
 
-    // Build the column expression, handling nested fields
-    let mut expr = col(&parts[0]);
+    // Build the column expression, handling nested fields.
+    let mut expr = Expr::Column(datafusion::common::Column::new_unqualified(
+        parts[0].clone(),
+    ));
     for part in &parts[1..] {
         expr = expr.field_newstyle(part);
     }
@@ -301,8 +299,26 @@ mod tests {
     use super::*;
 
     use arrow_schema::{Field, Schema as ArrowSchema};
+    use datafusion::common::Column;
     use datafusion_functions::core::expr_ext::FieldAccessor;
 
+    #[test]
+    fn test_field_path_to_expr_preserves_case_sensitive_root_column() {
+        let expr = field_path_to_expr("VECTOR").unwrap();
+
+        assert_eq!(expr, Expr::Column(Column::new_unqualified("VECTOR")));
+    }
+
+    #[test]
+    fn test_field_path_to_expr_preserves_case_sensitive_escaped_nested_path() {
+        let expr = field_path_to_expr("Parent.`Child.With.Dot`").unwrap();
+
+        assert_eq!(
+            expr,
+            Expr::Column(Column::new_unqualified("Parent")).field_newstyle("Child.With.Dot")
+        );
+    }
+
     #[test]
     fn test_resolve_large_utf8() {
         let arrow_schema = ArrowSchema::new(vec![Field::new("a", DataType::LargeUtf8, false)]);
diff --git a/rust/lance-datafusion/src/planner.rs b/rust/lance-datafusion/src/planner.rs
index 1e62cba42d8..f72e9fcfacb 100644
--- a/rust/lance-datafusion/src/planner.rs
+++ b/rust/lance-datafusion/src/planner.rs
@@ -72,10 +72,6 @@ impl CastListF16Udf {
 }
 
 impl ScalarUDFImpl for CastListF16Udf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn name(&self) -> &str {
         "_cast_list_f16"
     }
@@ -197,6 +193,13 @@ impl ContextProvider for LanceContextProvider {
         self.state.window_functions().get(name).cloned()
     }
 
+    fn get_higher_order_meta(
+        &self,
+        name: &str,
+    ) -> Option<Arc<datafusion::logical_expr::HigherOrderUDF>> {
+        self.state.higher_order_functions().get(name).cloned()
+    }
+
     fn get_function_meta(&self, f: &str) -> Option<Arc<ScalarUDF>> {
         match f {
             // TODO: cast should go thru CAST syntax instead of UDF
@@ -227,6 +230,14 @@ impl ContextProvider for LanceContextProvider {
         self.state.window_functions().keys().cloned().collect()
     }
 
+    fn higher_order_function_names(&self) -> Vec<String> {
+        self.state
+            .higher_order_functions()
+            .keys()
+            .cloned()
+            .collect()
+    }
+
     fn get_expr_planners(&self) -> &[Arc<dyn ExprPlanner>] {
         &self.expr_planners
     }
@@ -503,7 +514,7 @@ impl Planner {
             }
             _ => Err(Error::invalid_input(format!(
                 "Unsupported function args: {:?}",
-                &func.args
+                func.args
             ))),
         }
     }
@@ -747,10 +758,10 @@ impl Planner {
                 data_type, value, ..
             }) => {
                 let value = value.clone().into_string().expect_ok()?;
-                Ok(Expr::Cast(datafusion::logical_expr::Cast {
-                    expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)),
-                    data_type: self.parse_type(data_type)?,
-                }))
+                Ok(Expr::Cast(datafusion::logical_expr::Cast::new(
+                    Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)),
+                    self.parse_type(data_type)?,
+                )))
             }
             SQLExpr::IsFalse(expr) => Ok(Expr::IsFalse(Box::new(self.parse_sql_expr(expr)?))),
             SQLExpr::IsNotFalse(expr) => Ok(Expr::IsNotFalse(Box::new(self.parse_sql_expr(expr)?))),
@@ -783,7 +794,10 @@ impl Planner {
                 Box::new(self.parse_sql_expr(expr)?),
                 Box::new(self.parse_sql_expr(pattern)?),
                 match escape_char {
-                    Some(Value::SingleQuotedString(char)) => char.chars().next(),
+                    Some(ValueWithSpan {
+                        value: Value::SingleQuotedString(char),
+                        ..
+                    }) => char.chars().next(),
                     Some(value) => {
                         return Err(Error::invalid_input(format!(
                             "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}",
@@ -805,7 +819,10 @@ impl Planner {
                 Box::new(self.parse_sql_expr(expr)?),
                 Box::new(self.parse_sql_expr(pattern)?),
                 match escape_char {
-                    Some(Value::SingleQuotedString(char)) => char.chars().next(),
+                    Some(ValueWithSpan {
+                        value: Value::SingleQuotedString(char),
+                        ..
+                    }) => char.chars().next(),
                     Some(value) => {
                         return Err(Error::invalid_input(format!(
                             "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}",
@@ -838,15 +855,15 @@ impl Planner {
             } => match kind {
                 datafusion::sql::sqlparser::ast::CastKind::TryCast
                 | datafusion::sql::sqlparser::ast::CastKind::SafeCast => {
-                    Ok(Expr::TryCast(datafusion::logical_expr::TryCast {
-                        expr: Box::new(self.parse_sql_expr(expr)?),
-                        data_type: self.parse_type(data_type)?,
-                    }))
+                    Ok(Expr::TryCast(datafusion::logical_expr::TryCast::new(
+                        Box::new(self.parse_sql_expr(expr)?),
+                        self.parse_type(data_type)?,
+                    )))
                 }
-                _ => Ok(Expr::Cast(datafusion::logical_expr::Cast {
-                    expr: Box::new(self.parse_sql_expr(expr)?),
-                    data_type: self.parse_type(data_type)?,
-                })),
+                _ => Ok(Expr::Cast(datafusion::logical_expr::Cast::new(
+                    Box::new(self.parse_sql_expr(expr)?),
+                    self.parse_type(data_type)?,
+                ))),
             },
             SQLExpr::JsonAccess { .. } => Err(Error::invalid_input("JSON access is not supported")),
             SQLExpr::CompoundFieldAccess { root, access_chain } => {
@@ -991,9 +1008,10 @@ impl Planner {
 
         // DataFusion needs the coerce and simplify passes to be applied before
         // expressions can be handled by the physical planner.
-        let simplify_context = SimplifyContext::default()
+        let simplify_context = SimplifyContext::builder()
             .with_schema(df_schema.clone())
-            .with_query_execution_start_time(Some(Utc::now()));
+            .with_query_execution_start_time(Some(Utc::now()))
+            .build();
         let simplifier =
             datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
 
@@ -1062,13 +1080,9 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor {
                 self.columns.insert(path);
                 self.current_path.clear();
             }
-            Expr::ScalarFunction(udf) => {
-                if udf.name() == GetFieldFunc::default().name() {
-                    if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) {
-                        self.current_path.push_front(name.to_string())
-                    } else {
-                        self.current_path.clear();
-                    }
+            Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => {
+                if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) {
+                    self.current_path.push_front(name.to_string())
                 } else {
                     self.current_path.clear();
                 }
@@ -1084,7 +1098,6 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor {
 
 #[cfg(test)]
 mod tests {
-    use std::any::Any;
 
     use crate::logical_expr::ExprExt;
 
@@ -1204,10 +1217,6 @@ mod tests {
     }
 
     impl ScalarUDFImpl for StrictFloat64Udf {
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-
         fn name(&self) -> &str {
             "strict_float64"
         }
@@ -1609,7 +1618,7 @@ mod tests {
 
             match expr {
                 Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() {
-                    Expr::Cast(Cast { expr, data_type }) => {
+                    Expr::Cast(Cast { expr, field }) => {
                         match expr.as_ref() {
                             Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => {
                                 assert_eq!(value_str, expected_value_str);
@@ -1619,7 +1628,7 @@ mod tests {
                             }
                             _ => panic!("Expected cast to be applied to literal"),
                         }
-                        assert_eq!(data_type, expected_data_type);
+                        assert_eq!(field.data_type(), expected_data_type);
                     }
                     _ => panic!("Expected right to be a cast"),
                 },
@@ -1660,14 +1669,14 @@ mod tests {
 
             match expr {
                 Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() {
-                    Expr::Cast(Cast { expr, data_type }) => {
+                    Expr::Cast(Cast { expr, field }) => {
                         match expr.as_ref() {
                             Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => {
                                 assert_eq!(value_str, expected_value_str);
                             }
                             _ => panic!("Expected cast to be applied to literal"),
                         }
-                        assert_eq!(data_type, expected_data_type);
+                        assert_eq!(field.data_type(), expected_data_type);
                     }
                     _ => panic!("Expected right to be a cast"),
                 },
diff --git a/rust/lance-datafusion/src/spill.rs b/rust/lance-datafusion/src/spill.rs
index 8fa60c93ab6..7f4c45efa8b 100644
--- a/rust/lance-datafusion/src/spill.rs
+++ b/rust/lance-datafusion/src/spill.rs
@@ -9,13 +9,17 @@ use std::{
 
 use arrow::ipc::{reader::StreamReader, writer::StreamWriter};
 use arrow_array::RecordBatch;
-use arrow_schema::{ArrowError, Schema};
+use arrow_schema::{ArrowError, Schema, SchemaRef};
 use datafusion::{
-    execution::SendableRecordBatchStream, physical_plan::stream::RecordBatchStreamAdapter,
+    catalog::{TableProvider, streaming::StreamingTable},
+    execution::{SendableRecordBatchStream, TaskContext},
+    physical_plan::{stream::RecordBatchStreamAdapter, streaming::PartitionStream},
 };
 use datafusion_common::DataFusionError;
+use futures::StreamExt;
 use lance_arrow::memory::MemoryAccumulator;
 use lance_core::error::LanceOptionExt;
+use lance_core::utils::tempfile::TempDir;
 
 /// Start a spill of Arrow data to a file that can be read later multiple times.
 ///
@@ -60,6 +64,141 @@ pub fn create_replay_spill(
     (sender, receiver)
 }
 
+/// Wrap a one-shot [`SendableRecordBatchStream`] in a re-scannable [`TableProvider`].
+///
+/// The source is drained in the background into a replayable spill. Two properties
+/// keep this cheap for the common case:
+///
+/// - **Memory-first.** Up to `memory_limit` bytes are buffered in memory; the spill
+///   only touches disk once that budget is exceeded. A source that fits under the
+///   limit never hits the filesystem.
+/// - **Streaming replay.** A scan can start consuming batches as soon as they land,
+///   before the source has finished draining — the first reader is not blocked
+///   waiting for the whole source to buffer.
+///
+/// Each scan of the returned provider replays the full source, which is what makes a
+/// one-shot stream usable in the write retry loop.
+///
+/// The provider reports no statistics — the source size is not known until it has
+/// been fully drained — so callers that need source statistics (e.g. to drive join
+/// ordering) should prefer a materialized or file-backed provider instead.
+///
+/// # Examples
+///
+/// ```
+/// # use std::sync::Arc;
+/// # use arrow_array::{Int32Array, RecordBatch};
+/// # use arrow_schema::{DataType, Field, Schema};
+/// # use datafusion::execution::SendableRecordBatchStream;
+/// # use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+/// # use futures::TryStreamExt;
+/// # use lance_datafusion::exec::provider_to_stream;
+/// # use lance_datafusion::spill::spilling_table_provider;
+/// # #[tokio::main]
+/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+/// let batch =
+///     RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?;
+/// // A one-shot stream can only be consumed once.
+/// let source: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+///     schema.clone(),
+///     futures::stream::iter(vec![Ok(batch)]),
+/// ));
+///
+/// // Wrapping it makes it re-scannable: each scan replays the full source.
+/// let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?;
+/// let first: Vec<RecordBatch> = provider_to_stream(provider.clone()).await?.try_collect().await?;
+/// let second: Vec<RecordBatch> = provider_to_stream(provider).await?.try_collect().await?;
+/// assert_eq!(first.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
+/// assert_eq!(second.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
+/// # Ok(())
+/// # }
+/// ```
+pub async fn spilling_table_provider(
+    mut source: SendableRecordBatchStream,
+    memory_limit: usize,
+) -> Result<Arc<dyn TableProvider>, DataFusionError> {
+    let schema = source.schema();
+    let tmp_dir = tokio::task::spawn_blocking(TempDir::try_new)
+        .await
+        .map_err(|e| DataFusionError::Execution(format!("Failed to spawn temp dir task: {e}")))?
+        .map_err(|e| DataFusionError::Execution(format!("Failed to create temp dir: {e}")))?;
+    let tmp_path = tmp_dir.std_path().join("spill.arrows");
+    let (mut sender, receiver) = create_replay_spill(tmp_path, schema.clone(), memory_limit);
+
+    // Drain the one-shot source into the spill once, in the background. The spill
+    // tees to memory/disk so the first reader can consume batches as they arrive
+    // while later readers replay the complete source.
+    let drain_handle = tokio::task::spawn(async move {
+        let mut errored = false;
+        while let Some(res) = source.next().await {
+            match res {
+                Ok(batch) => {
+                    if let Err(e) = sender.write(batch).await {
+                        sender.send_error(e);
+                        errored = true;
+                        break;
+                    }
+                }
+                Err(e) => {
+                    sender.send_error(e);
+                    errored = true;
+                    break;
+                }
+            }
+        }
+        // Only finish on a clean drain. Calling finish() after an error would
+        // overwrite the original (replayable) error with a generic one, losing
+        // the source error's type (e.g. an external error from user code).
+        if !errored && let Err(err) = sender.finish().await {
+            sender.send_error(err);
+        }
+        sender
+    });
+
+    let partition = Arc::new(SpillPartition {
+        schema: schema.clone(),
+        receiver,
+        _tmp_dir: Arc::new(tmp_dir),
+        _drain_handle: Arc::new(drain_handle),
+    });
+    Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?))
+}
+
+/// A [`PartitionStream`] backed by a replayable spill.
+///
+/// Each call to [`PartitionStream::execute`] opens a fresh stream over the spill,
+/// so the partition can be scanned repeatedly. The spill file and the background
+/// task draining the source are kept alive for as long as this partition exists.
+struct SpillPartition {
+    schema: SchemaRef,
+    receiver: SpillReceiver,
+    // The spilled data lives in this temp dir; dropping it deletes the spill file.
+    _tmp_dir: Arc<TempDir>,
+    // Keeps the background drain task (which owns the `SpillSender`) alive. The
+    // `SpillSender` must outlive the readers or they error out, so we hold the
+    // handle rather than detaching it.
+    _drain_handle: Arc<tokio::task::JoinHandle<SpillSender>>,
+}
+
+impl std::fmt::Debug for SpillPartition {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SpillPartition")
+            .field("schema", &self.schema)
+            .finish()
+    }
+}
+
+impl PartitionStream for SpillPartition {
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
+    fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {
+        self.receiver.read()
+    }
+}
+
 #[derive(Clone)]
 pub struct SpillReceiver {
     status_receiver: tokio::sync::watch::Receiver<WriteStatus>,
diff --git a/rust/lance-datafusion/src/substrait.rs b/rust/lance-datafusion/src/substrait.rs
index 1c465fcae4a..f38e6b79cf9 100644
--- a/rust/lance-datafusion/src/substrait.rs
+++ b/rust/lance-datafusion/src/substrait.rs
@@ -164,6 +164,9 @@ fn remap_expr_references(expr: &mut Expression, mapping: &HashMap<usize, usize>)
         RexType::WindowFunction(_) | RexType::Subquery(_) => Err(Error::invalid_input(
             "Window functions or subqueries not allowed in filter expression",
         )),
+        RexType::Lambda(_) | RexType::LambdaInvocation(_) => Err(Error::invalid_input(
+            "Lambda expressions not allowed in filter expression",
+        )),
         // Pass through operators, nested children may have field references
         RexType::ScalarFunction(func) => {
             #[allow(deprecated)]
@@ -551,7 +554,7 @@ mod tests {
         },
         expression_reference::ExprType,
         extensions::{
-            SimpleExtensionDeclaration, SimpleExtensionUri, SimpleExtensionUrn,
+            SimpleExtensionDeclaration, SimpleExtensionUrn,
             simple_extension_declaration::{ExtensionFunction, MappingType},
         },
         function_argument::ArgType,
@@ -576,13 +579,6 @@ mod tests {
                 git_hash: "".to_string(),
                 producer: "unit-test".to_string(),
             }),
-            #[expect(deprecated)]
-            extension_uris: vec![
-                SimpleExtensionUri {
-                    extension_uri_anchor: 1,
-                    uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_comparison.yaml".to_string(),
-                }
-            ],
             extension_urns: vec![
                 SimpleExtensionUrn {
                     extension_urn_anchor: 1,
@@ -592,8 +588,6 @@ mod tests {
             extensions: vec![
                 SimpleExtensionDeclaration {
                     mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-                        #[expect(deprecated)]
-                        extension_uri_reference: 1,
                         extension_urn_reference: 1,
                         function_anchor: 1,
                         name: "lt".to_string(),
@@ -881,9 +875,7 @@ mod tests {
     fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration {
         SimpleExtensionDeclaration {
             mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-                #[allow(deprecated)]
-                extension_uri_reference: 1,
-                extension_urn_reference: 0,
+                extension_urn_reference: 1,
                 function_anchor: anchor,
                 name: name.to_string(),
             })),
@@ -919,10 +911,9 @@ mod tests {
                 git_hash: String::new(),
                 producer: "lance-test".to_string(),
             }),
-            #[allow(deprecated)]
-            extension_uris: vec![SimpleExtensionUri {
-                extension_uri_anchor: 1,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
+            extension_urns: vec![SimpleExtensionUrn {
+                extension_urn_anchor: 1,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
             }],
             extensions,
             relations: vec![PlanRel {
@@ -935,7 +926,6 @@ mod tests {
             }],
             advanced_extensions: None,
             expected_type_urls: vec![],
-            extension_urns: vec![],
             parameter_bindings: vec![],
             type_aliases: vec![],
         };
diff --git a/rust/lance-encoding/src/compression.rs b/rust/lance-encoding/src/compression.rs
index 9051e18e8b6..8d2d28f21b0 100644
--- a/rust/lance-encoding/src/compression.rs
+++ b/rust/lance-encoding/src/compression.rs
@@ -55,8 +55,8 @@ use crate::{
                 VariablePackedStructFieldKind,
             },
             rle::{
-                RleDecompressor, RleEncoder, RunLengthWidth, rle_encoded_size,
-                select_run_length_width,
+                RleChildDecompressor, RleDecompressor, RleEncoder, RunLengthWidth,
+                rle_encoded_size, select_run_length_width,
             },
             value::{ValueDecompressor, ValueEncoder},
         },
@@ -171,6 +171,7 @@ fn try_bss_for_mini_block(
 
 fn try_rle_for_mini_block(
     data: &FixedWidthDataBlock,
+    version: LanceFileVersion,
     params: &CompressionFieldParams,
     use_rle_v2: bool,
 ) -> Option<Box<dyn MiniBlockCompressor>> {
@@ -212,22 +213,59 @@ fn try_rle_for_mini_block(
         )
     };
 
-    if rle_bytes < raw_bytes {
-        #[cfg(feature = "bitpacking")]
+    let use_child_encodings = version.resolve() >= LanceFileVersion::V2_3;
+    let child_compression = if use_child_encodings {
+        rle_child_compression_config(params)
+    } else {
+        None
+    };
+    let use_child_bitpacking = use_child_encodings;
+    let rle_encoder = || {
+        if use_child_encodings {
+            RleEncoder::with_child_encoding(
+                run_length_width,
+                child_compression,
+                child_compression,
+                use_child_bitpacking,
+            )
+        } else {
+            RleEncoder::with_run_length_width(run_length_width)
+        }
+    };
+
+    #[cfg(feature = "bitpacking")]
+    let bitpack_bytes = estimate_inline_bitpacking_bytes(data).map(u128::from);
+    #[cfg(not(feature = "bitpacking"))]
+    let bitpack_bytes = None::<u128>;
+
+    let mut selected_rle_bytes = rle_bytes;
+    let should_estimate_child_size = use_child_encodings
+        && (child_compression.is_some() || cfg!(feature = "bitpacking"))
+        && (rle_bytes >= raw_bytes || bitpack_bytes.is_some_and(|bytes| bytes < rle_bytes));
+    if should_estimate_child_size {
+        selected_rle_bytes = rle_encoder().selected_payload_size(data).ok()?;
+    }
+
+    if selected_rle_bytes < raw_bytes {
+        if let Some(bitpack_bytes) = bitpack_bytes
+            && bitpack_bytes < selected_rle_bytes
         {
-            if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data)
-                && (bitpack_bytes as u128) < rle_bytes
-            {
-                return None;
-            }
+            return None;
         }
-        return Some(Box::new(RleEncoder::with_run_length_width(
-            run_length_width,
-        )));
+        return Some(Box::new(rle_encoder()));
     }
     None
 }
 
+fn rle_child_compression_config(params: &CompressionFieldParams) -> Option<CompressionConfig> {
+    let raw = params.compression.as_deref()?;
+    if matches!(raw, "none" | "fsst") {
+        return None;
+    }
+    let scheme = CompressionScheme::from_str(raw).ok()?;
+    Some(CompressionConfig::new(scheme, params.compression_level))
+}
+
 fn try_rle_for_block(
     data: &FixedWidthDataBlock,
     version: LanceFileVersion,
@@ -593,7 +631,7 @@ impl DefaultCompressionStrategy {
         }
 
         let base = try_bss_for_mini_block(data, params)
-            .or_else(|| try_rle_for_mini_block(data, params, self.use_rle_v2()))
+            .or_else(|| try_rle_for_mini_block(data, self.version, params, self.use_rle_v2()))
             .or_else(|| try_bitpack_for_mini_block(data))
             .unwrap_or_else(|| Box::new(ValueEncoder::default()));
 
@@ -757,6 +795,13 @@ impl CompressionStrategy for DefaultCompressionStrategy {
                 if (max_len > 32 * 1024 || per_value_requested)
                     && data_size >= FSST_LEAST_INPUT_SIZE as u64
                 {
+                    if compression == Some("zstd") {
+                        let config = CompressionConfig::new(
+                            CompressionScheme::Zstd,
+                            field_params.compression_level,
+                        );
+                        return Ok(Box::new(CompressedBufferEncoder::try_new(config)?));
+                    }
                     return Ok(Box::new(CompressedBufferEncoder::default()));
                 }
 
@@ -951,13 +996,10 @@ impl DecompressionStrategy for DefaultDecompressionStrategy {
                 // compression.
                 Ok(Box::new(ValueDecompressor::from_fsl(fsl)))
             }
-            Compression::Rle(rle) => {
-                let (bits_per_value, run_length_width) = validate_rle_compression(rle)?;
-                Ok(Box::new(RleDecompressor::with_run_length_width(
-                    bits_per_value,
-                    run_length_width,
-                )))
-            }
+            Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(
+                rle,
+                decompression_strategy,
+            )?)),
             Compression::ByteStreamSplit(bss) => {
                 let Compression::Flat(values) =
                     bss.values.as_ref().unwrap().compression.as_ref().unwrap()
@@ -1144,19 +1186,15 @@ impl DecompressionStrategy for DefaultDecompressionStrategy {
 
                 Ok(Box::new(general_decompressor))
             }
-            Compression::Rle(rle) => {
-                let (bits_per_value, run_length_width) = validate_rle_compression(rle)?;
-                Ok(Box::new(RleDecompressor::with_run_length_width(
-                    bits_per_value,
-                    run_length_width,
-                )))
-            }
+            Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(rle, self)?)),
             _ => todo!(),
         }
     }
 }
-/// Validates RLE compression format and extracts value and run length widths.
-fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunLengthWidth)> {
+fn create_rle_decompressor(
+    rle: &crate::format::pb21::Rle,
+    decompression_strategy: &dyn DecompressionStrategy,
+) -> Result<RleDecompressor> {
     let values = rle
         .values
         .as_ref()
@@ -1166,42 +1204,162 @@ fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunL
         .as_ref()
         .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths encoding"))?;
 
-    let values = values
-        .compression
-        .as_ref()
-        .ok_or_else(|| Error::invalid_input("RLE compression missing values compression"))?;
-    let Compression::Flat(values) = values else {
-        return Err(Error::invalid_input(
-            "RLE compression only supports flat values",
-        ));
-    };
+    let values = create_rle_child_decompressor(values, "values", decompression_strategy)?;
+    let run_lengths =
+        create_rle_child_decompressor(run_lengths, "run lengths", decompression_strategy)?;
 
-    let run_lengths = run_lengths
-        .compression
-        .as_ref()
-        .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths compression"))?;
-    let Compression::Flat(run_lengths) = run_lengths else {
-        return Err(Error::invalid_input(
-            "RLE compression only supports flat run lengths",
-        ));
-    };
-
-    if !matches!(values.bits_per_value, 8 | 16 | 32 | 64) {
+    if !matches!(values.bits_per_value(), 8 | 16 | 32 | 64) {
         return Err(Error::invalid_input(format!(
             "RLE compression only supports 8, 16, 32, or 64-bit values, got {}",
-            values.bits_per_value
+            values.bits_per_value()
         )));
     }
 
     let run_length_width =
-        RunLengthWidth::from_bits(run_lengths.bits_per_value).ok_or_else(|| {
+        RunLengthWidth::from_bits(run_lengths.bits_per_value()).ok_or_else(|| {
             Error::invalid_input(format!(
                 "RLE compression only supports 8, 16, or 32-bit run lengths, got {}",
-                run_lengths.bits_per_value
+                run_lengths.bits_per_value()
             ))
         })?;
 
-    Ok((values.bits_per_value, run_length_width))
+    if values.requires_num_values() && run_lengths.requires_num_values() {
+        return Err(Error::invalid_input(
+            "RLE values and run lengths child encodings cannot both require the run count",
+        ));
+    }
+
+    if values.is_identity() && run_lengths.is_identity() {
+        return Ok(RleDecompressor::with_run_length_width(
+            values.bits_per_value(),
+            run_length_width,
+        ));
+    }
+
+    Ok(RleDecompressor::with_child_decompressors(
+        values.bits_per_value(),
+        run_length_width,
+        values,
+        run_lengths,
+    ))
+}
+
+fn create_rle_child_decompressor(
+    encoding: &CompressiveEncoding,
+    role: &str,
+    decompression_strategy: &dyn DecompressionStrategy,
+) -> Result<RleChildDecompressor> {
+    let compression = encoding
+        .compression
+        .as_ref()
+        .ok_or_else(|| Error::invalid_input(format!("RLE {role} missing child compression")))?;
+    let (bits_per_value, requires_num_values, needs_decompressor) =
+        validate_rle_child_compression(compression, role)?;
+
+    if needs_decompressor {
+        Ok(RleChildDecompressor::block(
+            bits_per_value,
+            decompression_strategy.create_block_decompressor(encoding)?,
+            requires_num_values,
+        ))
+    } else {
+        Ok(RleChildDecompressor::flat(bits_per_value))
+    }
+}
+
+fn validate_rle_child_compression(
+    compression: &Compression,
+    role: &str,
+) -> Result<(u64, bool, bool)> {
+    match compression {
+        Compression::Flat(flat) => Ok((flat.bits_per_value, false, false)),
+        Compression::General(general) => {
+            general.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} general child missing compression config"
+                ))
+            })?;
+            let values = general.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!("RLE {role} general child missing inner encoding"))
+            })?;
+            let inner = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} general child missing inner compression"
+                ))
+            })?;
+            let (bits_per_value, requires_num_values) =
+                validate_rle_block_child_inner(inner, role)?;
+            Ok((bits_per_value, requires_num_values, true))
+        }
+        Compression::OutOfLineBitpacking(out_of_line) => {
+            let values = out_of_line.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values encoding"
+                ))
+            })?;
+            let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values compression"
+                ))
+            })?
+            else {
+                return Err(Error::invalid_input(format!(
+                    "RLE {role} bitpacking child only supports flat values"
+                )));
+            };
+            Ok((out_of_line.uncompressed_bits_per_value, true, true))
+        }
+        other => Err(Error::invalid_input(format!(
+            "RLE {role} only supports flat, general, or out-of-line bitpacking child encodings, got {}",
+            compression_name(other)
+        ))),
+    }
+}
+
+fn validate_rle_block_child_inner(compression: &Compression, role: &str) -> Result<(u64, bool)> {
+    match compression {
+        Compression::Flat(flat) => Ok((flat.bits_per_value, false)),
+        Compression::OutOfLineBitpacking(out_of_line) => {
+            let values = out_of_line.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values encoding"
+                ))
+            })?;
+            let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values compression"
+                ))
+            })?
+            else {
+                return Err(Error::invalid_input(format!(
+                    "RLE {role} bitpacking child only supports flat values"
+                )));
+            };
+            Ok((out_of_line.uncompressed_bits_per_value, true))
+        }
+        other => Err(Error::invalid_input(format!(
+            "RLE {role} general child only supports flat or out-of-line bitpacking inner encodings, got {}",
+            compression_name(other)
+        ))),
+    }
+}
+
+fn compression_name(compression: &Compression) -> &'static str {
+    match compression {
+        Compression::Flat(_) => "flat",
+        Compression::Variable(_) => "variable",
+        Compression::Fsst(_) => "fsst",
+        Compression::OutOfLineBitpacking(_) => "out-of-line bitpacking",
+        Compression::InlineBitpacking(_) => "inline bitpacking",
+        Compression::General(_) => "general",
+        Compression::Constant(_) => "constant",
+        Compression::Dictionary(_) => "dictionary",
+        Compression::ByteStreamSplit(_) => "byte stream split",
+        Compression::PackedStruct(_) => "packed struct",
+        Compression::FixedSizeList(_) => "fixed-size list",
+        Compression::VariablePackedStruct(_) => "variable packed struct",
+        Compression::Rle(_) => "rle",
+    }
 }
 
 #[cfg(test)]
@@ -1307,6 +1465,20 @@ mod tests {
         run_lengths.bits_per_value
     }
 
+    fn expect_rle_encoding(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle {
+        match encoding.compression.as_ref().unwrap() {
+            Compression::Rle(rle) => rle,
+            Compression::General(general) => {
+                let inner = general.values.as_ref().unwrap();
+                let Compression::Rle(rle) = inner.compression.as_ref().unwrap() else {
+                    panic!("expected wrapped RLE encoding");
+                };
+                rle
+            }
+            other => panic!("expected RLE encoding, got {}", compression_name(other)),
+        }
+    }
+
     fn create_variable_width_block(
         bits_per_offset: u8,
         num_values: u64,
@@ -1727,6 +1899,30 @@ mod tests {
         );
     }
 
+    #[test]
+    #[cfg(feature = "zstd")]
+    fn test_compression_level_honored_for_large_per_value() {
+        let mut params = CompressionParams::new();
+        params.columns.insert(
+            "html".to_string(),
+            CompressionFieldParams {
+                compression: Some("zstd".to_string()),
+                compression_level: Some(19),
+                ..Default::default()
+            },
+        );
+        let strategy = DefaultCompressionStrategy::with_params(params);
+        let field = create_test_field("html", DataType::Utf8);
+        let large = create_variable_width_block(32, 64, 40 * 1024);
+
+        let per_value = strategy.create_per_value(&field, &large).unwrap();
+        let debug = format!("{per_value:?}");
+        assert!(
+            debug.contains("ZstdBufferCompressor") && debug.contains("compression_level: 19"),
+            "expected zstd level 19 to reach the per-value compressor, got: {debug}"
+        );
+    }
+
     #[test]
     fn test_parameter_merge_priority() {
         let mut params = CompressionParams::new();
@@ -2004,6 +2200,154 @@ mod tests {
         assert_eq!(rle_run_length_bits(&encoding), 8);
     }
 
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_released_versions_keep_flat_children_when_compression_requested() {
+        for version in [LanceFileVersion::V2_1, LanceFileVersion::V2_2] {
+            let mut params = CompressionParams::new();
+            params.columns.insert(
+                "dict_indices".to_string(),
+                CompressionFieldParams {
+                    compression: Some(
+                        if cfg!(feature = "lz4") { "lz4" } else { "zstd" }.to_string(),
+                    ),
+                    rle_threshold: Some(1.0),
+                    bss: Some(BssMode::Off),
+                    ..Default::default()
+                },
+            );
+            let strategy = DefaultCompressionStrategy::with_params(params).with_version(version);
+            let field = create_test_field("dict_indices", DataType::UInt32);
+
+            let mut values = Vec::with_capacity(8192 * 4);
+            for value in 0..8192u32 {
+                values.extend(std::iter::repeat_n(value, 4));
+            }
+            let mut data = FixedWidthDataBlock {
+                bits_per_value: 32,
+                data: LanceBuffer::reinterpret_vec(values),
+                num_values: 8192 * 4,
+                block_info: BlockInfo::default(),
+            };
+            data.compute_stat();
+            let data = DataBlock::FixedWidth(data);
+
+            let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+            let (_compressed, encoding) = compressor.compress(data).unwrap();
+            let rle = expect_rle_encoding(&encoding);
+
+            assert!(
+                matches!(
+                    rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+                    Compression::Flat(_)
+                ),
+                "version={version}"
+            );
+            assert!(
+                matches!(
+                    rle.run_lengths
+                        .as_ref()
+                        .unwrap()
+                        .compression
+                        .as_ref()
+                        .unwrap(),
+                    Compression::Flat(_)
+                ),
+                "version={version}"
+            );
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_strategy_bitpacks_child_values_when_smaller() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+
+        let mut values = Vec::with_capacity(8192 * 4);
+        for value in 0..8192 {
+            values.extend(std::iter::repeat_n(value, 4));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 8192 * 4,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(debug_str.contains("RleEncoder"));
+
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        let Compression::Rle(rle) = encoding.compression.as_ref().unwrap() else {
+            panic!("expected RLE encoding");
+        };
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            Compression::Flat(_)
+        ));
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_keeps_child_bitpacked_rle_when_smaller_than_inline_bitpacking() {
+        let field = create_test_field("int_score", DataType::UInt64);
+
+        let mut values = Vec::with_capacity(8192 * 8);
+        for run_idx in 0..8192 {
+            let value = match run_idx % 3 {
+                0 => 3u64,
+                1 => 4u64,
+                _ => 5u64,
+            };
+            values.extend(std::iter::repeat_n(value, 8));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 64,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 8192 * 8,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(
+            debug_str.contains("RleEncoder"),
+            "expected RLE to beat inline bitpacking after child selection, got: {debug_str}"
+        );
+
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        let rle = expect_rle_encoding(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            Compression::Flat(_)
+        ));
+    }
+
     #[test]
     fn test_field_metadata_override_params() {
         // Set up params with one configuration
diff --git a/rust/lance-encoding/src/constants.rs b/rust/lance-encoding/src/constants.rs
index c95b587a532..0bd31d676cc 100644
--- a/rust/lance-encoding/src/constants.rs
+++ b/rust/lance-encoding/src/constants.rs
@@ -55,6 +55,8 @@ pub const STRUCTURAL_ENCODING_META_KEY: &str = "lance-encoding:structural-encodi
 pub const STRUCTURAL_ENCODING_MINIBLOCK: &str = "miniblock";
 /// Value for fullzip structural encoding
 pub const STRUCTURAL_ENCODING_FULLZIP: &str = "fullzip";
+/// Value for sparse structural encoding
+pub const STRUCTURAL_ENCODING_SPARSE: &str = "sparse";
 
 // Byte stream split metadata keys
 /// Metadata key for byte stream split encoding configuration
diff --git a/rust/lance-encoding/src/decoder.rs b/rust/lance-encoding/src/decoder.rs
index 55340d09c94..108a848db64 100644
--- a/rust/lance-encoding/src/decoder.rs
+++ b/rust/lance-encoding/src/decoder.rs
@@ -1949,8 +1949,7 @@ impl StructuralBatchDecodeStream {
                     next_task.into_batch(emitted_batch_size_warning)?
                 };
                 let num_rows = batch.num_rows() as u64;
-                if num_rows > 0 {
-                    let bpr = data_size / num_rows;
+                if let Some(bpr) = data_size.checked_div(num_rows) {
                     let prev = bytes_per_row_feedback.load(Ordering::Relaxed);
                     let next = if prev == 0 || bpr >= prev {
                         // First batch or actual size is larger than estimate:
@@ -2718,8 +2717,9 @@ pub trait DecodeArrayTask: Send {
 
 impl DecodeArrayTask for Box<dyn StructuralDecodeArrayTask> {
     fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
-        StructuralDecodeArrayTask::decode(*self)
-            .map(|decoded_array| (decoded_array.array, decoded_array.data_size))
+        let decoded_array = StructuralDecodeArrayTask::decode(*self)?;
+        decoded_array.repdef.ensure_exhausted()?;
+        Ok((decoded_array.array, decoded_array.data_size))
     }
 }
 
@@ -2741,10 +2741,7 @@ impl NextDecodeTask {
     // suggesting the user try a smaller batch size.
     #[instrument(name = "task_to_batch", level = "debug", skip_all)]
     fn into_batch(self, emitted_batch_size_warning: Arc<Once>) -> Result<(RecordBatch, u64)> {
-        let (struct_arr, data_size) = self
-            .task
-            .decode()
-            .map_err(|e| Error::internal(format!("Error decoding batch: {}", e)))?;
+        let (struct_arr, data_size) = self.task.decode()?;
         let batch = RecordBatch::from(struct_arr.as_struct());
         if data_size > BATCH_SIZE_BYTES_WARNING {
             emitted_batch_size_warning.call_once(|| {
@@ -2984,6 +2981,25 @@ mod tests {
         }
     }
 
+    struct InvalidInputDecodeTask;
+
+    impl DecodeArrayTask for InvalidInputDecodeTask {
+        fn decode(self: Box<Self>) -> Result<(ArrayRef, u64)> {
+            Err(Error::invalid_input_source("malformed sparse page".into()))
+        }
+    }
+
+    #[test]
+    fn next_decode_task_preserves_invalid_input_errors() {
+        let err = NextDecodeTask {
+            task: Box::new(InvalidInputDecodeTask),
+            num_rows: 0,
+        }
+        .into_batch(Arc::new(Once::new()))
+        .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+    }
+
     #[test]
     fn test_read_zero_dimension_fsl_errors_instead_of_panicking() {
         // Simulates reading a column whose stored schema declares a
diff --git a/rust/lance-encoding/src/encodings/logical/blob.rs b/rust/lance-encoding/src/encodings/logical/blob.rs
index cad2112bafe..d1798b1ef69 100644
--- a/rust/lance-encoding/src/encodings/logical/blob.rs
+++ b/rust/lance-encoding/src/encodings/logical/blob.rs
@@ -267,16 +267,11 @@ impl FieldEncoder for BlobV2StructuralEncoder {
         &mut self,
         array: ArrayRef,
         external_buffers: &mut OutOfLineBuffers,
-        mut repdef: RepDefBuilder,
+        repdef: RepDefBuilder,
         row_number: u64,
         num_rows: u64,
     ) -> Result<Vec<EncodeTask>> {
         let struct_arr = array.as_struct();
-        if let Some(validity) = struct_arr.nulls() {
-            repdef.add_validity_bitmap(validity.clone());
-        } else {
-            repdef.add_no_null(struct_arr.len());
-        }
 
         let kind_col = struct_arr
             .column_by_name("kind")
@@ -403,7 +398,7 @@ impl FieldEncoder for BlobV2StructuralEncoder {
         let descriptor_array = Arc::new(StructArray::try_new(
             BLOB_V2_DESC_FIELDS.clone(),
             children,
-            None,
+            struct_arr.nulls().cloned(),
         )?) as ArrayRef;
 
         self.descriptor_encoder.maybe_encode(
@@ -535,6 +530,32 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn test_blob_round_trip_empty_values() {
+        // Empty values share size == 0 with nulls in the descriptor layout
+        // and schedule no read; each must decode to zero-length bytes without
+        // consuming the read result of a following non-empty blob. Empties
+        // are placed before payloads so a misassignment corrupts the output
+        // instead of only exhausting the read iterator.
+        let blob_metadata =
+            HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]);
+
+        let val1: &[u8] = &vec![1u8; 1024];
+        let val2: &[u8] = &vec![2u8; 10240];
+        let empty: &[u8] = &[];
+        let array = Arc::new(LargeBinaryArray::from(vec![
+            Some(empty),
+            Some(val1),
+            None,
+            Some(empty),
+            Some(val2),
+            None,
+            Some(empty),
+        ]));
+
+        check_round_trip_encoding_of_data(vec![array], &TestCases::default(), blob_metadata).await;
+    }
+
     #[tokio::test]
     async fn test_blob_v2_external_round_trip() {
         let blob_metadata = HashMap::from([(
diff --git a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
index 9e8e3e109ea..1308288f226 100644
--- a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
+++ b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs
@@ -65,7 +65,7 @@ impl FieldEncoder for FixedSizeListStructuralEncoder {
         } else {
             deep_copy_nulls(array.nulls())
         };
-        repdef.add_fsl(validity.clone(), dimension, num_rows as usize);
+        repdef.add_fsl(validity.clone(), dimension, fsl_arr.len());
 
         // FSL forces child elements to exist even under null rows. Normalize any
         // nested lists under null FSL rows to null empty lists.
@@ -255,7 +255,7 @@ impl StructuralDecodeArrayTask for StructuralFixedSizeListDecodeTask {
         match &self.data_type {
             DataType::FixedSizeList(child_field, dimension) => {
                 let num_rows = self.num_rows as usize;
-                let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize);
+                let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize)?;
                 let fsl_array = arrow_array::FixedSizeListArray::try_new(
                     child_field.clone(),
                     *dimension,
diff --git a/rust/lance-encoding/src/encodings/logical/list.rs b/rust/lance-encoding/src/encodings/logical/list.rs
index 250eb476671..c9bdcc0bc87 100644
--- a/rust/lance-encoding/src/encodings/logical/list.rs
+++ b/rust/lance-encoding/src/encodings/logical/list.rs
@@ -344,6 +344,7 @@ mod tests {
                     }
                 }
                 crate::format::pb21::page_layout::Layout::BlobLayout(_) => {}
+                crate::format::pb21::page_layout::Layout::SparseLayout(_) => {}
             }
         }
 
diff --git a/rust/lance-encoding/src/encodings/logical/map.rs b/rust/lance-encoding/src/encodings/logical/map.rs
index b5172d8c189..be153030105 100644
--- a/rust/lance-encoding/src/encodings/logical/map.rs
+++ b/rust/lance-encoding/src/encodings/logical/map.rs
@@ -223,7 +223,8 @@ impl StructuralDecodeArrayTask for StructuralMapDecodeTask {
             .clone();
 
         // Build the MapArray from offsets, entries, validity, and keys_sorted
-        let map_array = MapArray::new(entries_field, offsets, entries, validity, keys_sorted);
+        let map_array = MapArray::try_new(entries_field, offsets, entries, validity, keys_sorted)
+            .map_err(|error| Error::invalid_input_source(error.to_string().into()))?;
 
         Ok(DecodedArray {
             array: Arc::new(map_array),
@@ -244,7 +245,13 @@ mod tests {
     use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
     use arrow_schema::{DataType, Field, Fields};
 
+    use crate::decoder::{DecodedArray, StructuralDecodeArrayTask};
     use crate::encoder::{ColumnIndexSequence, EncodingOptions, default_encoding_strategy};
+    use crate::encodings::logical::primitive::sparse::{
+        SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan,
+        SparseValidityMeaning, SparseValiditySet,
+    };
+    use crate::repdef::{CompositeRepDefUnraveler, RepDefUnraveler};
     use crate::{
         testing::{TestCases, check_round_trip_encoding_of_data},
         version::LanceFileVersion,
@@ -252,6 +259,8 @@ mod tests {
     use arrow_schema::Field as ArrowField;
     use lance_core::datatypes::Field as LanceField;
 
+    use super::StructuralMapDecodeTask;
+
     fn make_map_type(key_type: DataType, value_type: DataType) -> DataType {
         // Note: Arrow MapBuilder uses "keys" and "values" as field names (plural)
         let entries = Field::new(
@@ -265,6 +274,70 @@ mod tests {
         DataType::Map(Arc::new(entries), false)
     }
 
+    #[derive(Debug)]
+    struct StaticMapEntriesTask {
+        entries: StructArray,
+        repdef: CompositeRepDefUnraveler,
+    }
+
+    impl StructuralDecodeArrayTask for StaticMapEntriesTask {
+        fn decode(self: Box<Self>) -> lance_core::Result<DecodedArray> {
+            let Self { entries, repdef } = *self;
+            Ok(DecodedArray {
+                array: Arc::new(entries),
+                repdef,
+                data_size: 0,
+            })
+        }
+    }
+
+    #[test]
+    fn malformed_sparse_map_entries_return_invalid_input() {
+        let entry_fields = Fields::from(vec![
+            Field::new("keys", DataType::Int32, false),
+            Field::new("values", DataType::Int32, true),
+        ]);
+        let entries = StructArray::try_new(
+            entry_fields.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1])),
+                Arc::new(Int32Array::from(vec![2])),
+            ],
+            Some(NullBuffer::from(vec![false])),
+        )
+        .unwrap();
+        let validity = SparseValiditySet {
+            meaning: SparseValidityMeaning::NullPositions,
+            positions: SparsePositionSet::Empty,
+        };
+        let plan = SparseStructuralPlan {
+            layers: vec![SparseStructuralLayerPlan::List {
+                num_slots: 1,
+                num_child_slots: 1,
+                non_empty_positions: SparsePositionSet::All { len: 1 },
+                counts: SparseCountSet::Constant { value: 1, len: 1 },
+                validity,
+            }],
+            num_items: 1,
+            num_visible_items: 1,
+        };
+        let child_task = StaticMapEntriesTask {
+            entries,
+            repdef: CompositeRepDefUnraveler::new(vec![RepDefUnraveler::new_sparse(plan)]),
+        };
+        let map_type = DataType::Map(
+            Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)),
+            false,
+        );
+
+        let Err(err) =
+            Box::new(StructuralMapDecodeTask::new(Box::new(child_task), map_type)).decode()
+        else {
+            panic!("expected malformed map entries to be rejected");
+        };
+        assert!(matches!(err, lance_core::Error::InvalidInput { .. }));
+    }
+
     #[test_log::test(tokio::test)]
     async fn test_simple_map() {
         // Create a simple Map<String, Int32>
diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs
index 8b6eed5d757..c13ef2f3034 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive.rs
@@ -15,6 +15,7 @@ use std::{
 use crate::{
     constants::{
         STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK,
+        STRUCTURAL_ENCODING_SPARSE,
     },
     data::DictionaryDataBlock,
     encodings::logical::primitive::blob::{BlobDescriptionPageScheduler, BlobPageScheduler},
@@ -36,7 +37,7 @@ use lance_core::{
     error::{Error, LanceOptionExt},
     utils::bit::pad_bytes,
 };
-use log::trace;
+use log::{debug, trace};
 
 use crate::encodings::logical::primitive::miniblock::MiniBlockChunk;
 use crate::utils::bytepack::ByteUnpacker;
@@ -58,7 +59,7 @@ use crate::{
 use crate::{
     repdef::{
         CompositeRepDefUnraveler, ControlWordIterator, ControlWordParser, DefinitionInterpretation,
-        RepDefSlicer, SerializedRepDefs, StructuralPagePlan, build_control_word_iterator,
+        MiniBlockRepDefBudget, RepDefSlicer, SerializedRepDefs, build_control_word_iterator,
     },
     utils::accumulation::AccumulationQueue,
 };
@@ -88,10 +89,15 @@ use crate::{
 };
 
 pub mod blob;
+mod chunk_index;
 pub mod constant;
 pub mod dict;
 pub mod fullzip;
+mod layout;
 pub mod miniblock;
+pub(crate) mod sparse;
+
+use chunk_index::{ItemCounts, MiniBlockChunkIndex, PrefixSums, RowMapping, parse_nested_rep};
 
 const FILL_BYTE: u8 = 0xFE;
 const DEFAULT_DICT_DIVISOR: u64 = 2;
@@ -1206,121 +1212,18 @@ struct MiniBlockSchedulerDictionary {
     num_dictionary_items: u64,
 }
 
-/// Individual block metadata within a MiniBlock repetition index.
-#[derive(Debug)]
-struct MiniBlockRepIndexBlock {
-    // The index of the first row that starts after the beginning of this block.  If the block
-    // has a preamble this will be the row after the preamble.  If the block is entirely preamble
-    // then this will be a row that starts in some future block.
-    first_row: u64,
-    // The number of rows in the block, including the trailer but not the preamble.
-    // Can be 0 if the block is entirely preamble
-    starts_including_trailer: u64,
-    // Whether the block has a preamble
-    has_preamble: bool,
-    // Whether the block has a trailer
-    has_trailer: bool,
-}
-
-impl DeepSizeOf for MiniBlockRepIndexBlock {
-    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
-        0
-    }
-}
-
-/// Repetition index for MiniBlock encoding.
-///
-/// Stores block-level offset information to enable efficient random
-/// access to nested data structures within mini-blocks.
-#[derive(Debug)]
-struct MiniBlockRepIndex {
-    blocks: Vec<MiniBlockRepIndexBlock>,
-}
-
-impl DeepSizeOf for MiniBlockRepIndex {
-    fn deep_size_of_children(&self, context: &mut Context) -> usize {
-        self.blocks.deep_size_of_children(context)
-    }
-}
-
-impl MiniBlockRepIndex {
-    /// Decode repetition index from chunk metadata using default values.
-    ///
-    /// This creates a repetition index where each chunk has no partial values
-    /// and no trailers, suitable for simple sequential data layouts.
-    pub fn default_from_chunks(chunks: &[ChunkMeta]) -> Self {
-        let mut blocks = Vec::with_capacity(chunks.len());
-        let mut offset: u64 = 0;
-
-        for c in chunks {
-            blocks.push(MiniBlockRepIndexBlock {
-                first_row: offset,
-                starts_including_trailer: c.num_values,
-                has_preamble: false,
-                has_trailer: false,
-            });
-
-            offset += c.num_values;
-        }
-
-        Self { blocks }
-    }
-
-    /// Decode repetition index from raw bytes in little-endian format.
-    ///
-    /// The bytes should contain u64 values arranged in groups of `stride` elements,
-    /// where the first two values of each group represent ends_count and partial_count.
-    /// Returns an empty index if no bytes are provided.
-    pub fn decode_from_bytes(rep_bytes: &[u8], stride: usize) -> Self {
-        // Convert bytes to u64 slice, handling alignment automatically
-        let buffer = crate::buffer::LanceBuffer::from(rep_bytes.to_vec());
-        let u64_slice = buffer.borrow_to_typed_slice::<u64>();
-        let n = u64_slice.len() / stride;
-
-        let mut blocks = Vec::with_capacity(n);
-        let mut chunk_has_preamble = false;
-        let mut offset: u64 = 0;
-
-        // Extract first two values from each block: ends_count and partial_count
-        for i in 0..n {
-            let base_idx = i * stride;
-            let ends = u64_slice[base_idx];
-            let partial = u64_slice[base_idx + 1];
-
-            let has_trailer = partial > 0;
-            // Convert branches to arithmetic for better compiler optimization
-            let starts_including_trailer =
-                ends + (has_trailer as u64) - (chunk_has_preamble as u64);
-
-            blocks.push(MiniBlockRepIndexBlock {
-                first_row: offset,
-                starts_including_trailer,
-                has_preamble: chunk_has_preamble,
-                has_trailer,
-            });
-
-            chunk_has_preamble = has_trailer;
-            offset += starts_including_trailer;
-        }
-
-        Self { blocks }
-    }
-}
-
 /// State that is loaded once and cached for future lookups
 #[derive(Debug)]
 struct MiniBlockCacheableState {
-    /// Metadata that describes each chunk in the page
-    chunk_meta: Vec<ChunkMeta>,
-    /// The decoded repetition index
-    rep_index: MiniBlockRepIndex,
+    /// Compact per-chunk index (byte ranges + row/item mapping) for the page
+    chunk_index: MiniBlockChunkIndex,
     /// The dictionary for the page, if any
     dictionary: Option<Arc<DataBlock>>,
 }
 
 impl DeepSizeOf for MiniBlockCacheableState {
     fn deep_size_of_children(&self, context: &mut Context) -> usize {
-        self.rep_index.deep_size_of_children(context)
+        self.chunk_index.deep_size_of_children(context)
             + self
                 .dictionary
                 .as_ref()
@@ -1465,19 +1368,14 @@ impl MiniBlockScheduler {
     }
 
     fn lookup_chunks(&self, chunk_indices: &[usize]) -> Vec<LoadedChunk> {
-        let page_meta = self.page_meta.as_ref().unwrap();
+        let chunk_index = &self.page_meta.as_ref().unwrap().chunk_index;
         chunk_indices
             .iter()
-            .map(|&chunk_idx| {
-                let chunk_meta = &page_meta.chunk_meta[chunk_idx];
-                let bytes_start = chunk_meta.offset_bytes;
-                let bytes_end = bytes_start + chunk_meta.chunk_size_bytes;
-                LoadedChunk {
-                    byte_range: bytes_start..bytes_end,
-                    items_in_chunk: chunk_meta.num_values,
-                    chunk_idx,
-                    data: LanceBuffer::empty(),
-                }
+            .map(|&chunk_idx| LoadedChunk {
+                byte_range: chunk_index.byte_range(chunk_idx),
+                items_in_chunk: chunk_index.items_in_chunk(chunk_idx),
+                chunk_idx,
+                data: LanceBuffer::empty(),
             })
             .collect()
     }
@@ -1582,9 +1480,12 @@ impl ChunkInstructions {
     //
     // The output will be a set of `ChunkInstructions` which tell us how to read from the chunks
     fn schedule_instructions(
-        rep_index: &MiniBlockRepIndex,
+        chunk_index: &MiniBlockChunkIndex,
         user_ranges: &[Range<u64>],
     ) -> Vec<Self> {
+        // Bind the per-page chunk count once; re-deriving it each iteration
+        // costs a width match plus a length read.
+        let num_chunks = chunk_index.num_chunks();
         // This is an in-exact capacity guess but pretty good.  The actual capacity can be
         // smaller if instructions are merged.  It can be larger if there are multiple instructions
         // per row which can happen with lists.
@@ -1596,43 +1497,30 @@ impl ChunkInstructions {
 
             // Need to find the first chunk with a first row >= user_range.start.  If there are
             // multiple chunks with the same first row we need to take the first one.
-            let mut block_index = match rep_index
-                .blocks
-                .binary_search_by_key(&user_range.start, |block| block.first_row)
-            {
-                Ok(idx) => {
-                    // Slightly tricky case, we may need to walk backwards a bit to make sure we
-                    // are grabbing first eligible chunk
-                    let mut idx = idx;
-                    while idx > 0 && rep_index.blocks[idx - 1].first_row == user_range.start {
-                        idx -= 1;
-                    }
-                    idx
-                }
-                // Easy case.  idx is greater, and idx - 1 is smaller, so idx - 1 contains the start
-                Err(idx) => idx - 1,
-            };
+            let mut block_index = chunk_index.find_chunk(user_range.start);
 
-            let mut to_skip = user_range.start - rep_index.blocks[block_index].first_row;
+            let mut to_skip = user_range.start - chunk_index.first_row(block_index);
 
             while rows_needed > 0 || need_preamble {
                 // Check if we've gone past the last block (should not happen)
-                if block_index >= rep_index.blocks.len() {
+                if block_index >= num_chunks {
                     log::warn!(
-                        "schedule_instructions inconsistency: block_index >= rep_index.blocks.len(), exiting early"
+                        "schedule_instructions inconsistency: block_index >= num_chunks, exiting early"
                     );
                     break;
                 }
 
-                let chunk = &rep_index.blocks[block_index];
-                let rows_avail = chunk.starts_including_trailer.saturating_sub(to_skip);
+                let starts_including_trailer = chunk_index.rows_in_chunk(block_index);
+                let has_preamble = chunk_index.has_preamble(block_index);
+                let has_trailer = chunk_index.has_trailer(block_index);
+                let rows_avail = starts_including_trailer.saturating_sub(to_skip);
 
                 // Handle blocks that are entirely preamble (rows_avail = 0)
                 // These blocks have no rows to take but may have a preamble we need
                 // We only look for preamble if to_skip == 0 (we're not skipping rows)
                 if rows_avail == 0 && to_skip == 0 {
                     // Only process if this chunk has a preamble we need
-                    if chunk.has_preamble && need_preamble {
+                    if has_preamble && need_preamble {
                         chunk_instructions.push(Self {
                             chunk_idx: block_index,
                             preamble: PreambleAction::Take,
@@ -1641,14 +1529,12 @@ impl ChunkInstructions {
                             // We still need to look at has_trailer to distinguish between "all preamble
                             // and row ends at end of chunk" and "all preamble and row bleeds into next
                             // chunk".  Both cases will have 0 rows available.
-                            take_trailer: chunk.has_trailer,
+                            take_trailer: has_trailer,
                         });
                         // Only set need_preamble = false if the chunk has at least one row,
                         // Or we are reaching the last block,
                         // Otherwise, the chunk is entirely preamble and we need the next chunk's preamble too
-                        if chunk.starts_including_trailer > 0
-                            || block_index == rep_index.blocks.len() - 1
-                        {
+                        if starts_including_trailer > 0 || block_index == num_chunks - 1 {
                             need_preamble = false;
                         }
                     }
@@ -1663,7 +1549,7 @@ impl ChunkInstructions {
                 if rows_avail == 0 && to_skip > 0 {
                     // This block doesn't have enough rows to skip, move to next block
                     // Adjust to_skip by the number of rows in this block
-                    to_skip -= chunk.starts_including_trailer;
+                    to_skip -= starts_including_trailer;
                     block_index += 1;
                     continue;
                 }
@@ -1672,7 +1558,7 @@ impl ChunkInstructions {
                 rows_needed -= rows_to_take;
 
                 let mut take_trailer = false;
-                let preamble = if chunk.has_preamble {
+                let preamble = if has_preamble {
                     if need_preamble {
                         PreambleAction::Take
                     } else {
@@ -1683,7 +1569,7 @@ impl ChunkInstructions {
                 };
 
                 // Are we taking the trailer?  If so, make sure we mark that we need the preamble
-                if rows_to_take == rows_avail && chunk.has_trailer {
+                if rows_to_take == rows_avail && has_trailer {
                     take_trailer = true;
                     need_preamble = true;
                 } else {
@@ -1707,25 +1593,33 @@ impl ChunkInstructions {
         // are _adjacent_ (i.e. don't merge "take first row of chunk 0" and "take third row of chunk 0" into "take 2
         // rows of chunk 0 starting at 0")
         if user_ranges.len() > 1 {
-            // TODO: Could probably optimize this allocation away
-            let mut merged_instructions = Vec::with_capacity(chunk_instructions.len());
-            let mut instructions_iter = chunk_instructions.into_iter();
-            merged_instructions.push(instructions_iter.next().unwrap());
-            for instruction in instructions_iter {
-                let last = merged_instructions.last_mut().unwrap();
-                if last.chunk_idx == instruction.chunk_idx
-                    && last.rows_to_take + last.rows_to_skip == instruction.rows_to_skip
-                {
-                    last.rows_to_take += instruction.rows_to_take;
-                    last.take_trailer |= instruction.take_trailer;
+            // Merge adjacent instructions in place.  `write` indexes the last
+            // retained instruction; each following instruction is either folded
+            // into it (contiguous within the same chunk) or compacted forward.
+            let mut write = 0;
+            for read in 1..chunk_instructions.len() {
+                let merges = {
+                    let last = &chunk_instructions[write];
+                    let candidate = &chunk_instructions[read];
+                    last.chunk_idx == candidate.chunk_idx
+                        && last.rows_to_take + last.rows_to_skip == candidate.rows_to_skip
+                };
+                if merges {
+                    let rows_to_take = chunk_instructions[read].rows_to_take;
+                    let take_trailer = chunk_instructions[read].take_trailer;
+                    let last = &mut chunk_instructions[write];
+                    last.rows_to_take += rows_to_take;
+                    last.take_trailer |= take_trailer;
                 } else {
-                    merged_instructions.push(instruction);
+                    write += 1;
+                    if write != read {
+                        chunk_instructions.swap(write, read);
+                    }
                 }
             }
-            merged_instructions
-        } else {
-            chunk_instructions
+            chunk_instructions.truncate(write + 1);
         }
+        chunk_instructions
     }
 
     fn drain_from_instruction(
@@ -1836,6 +1730,129 @@ impl<'a> Iterator for WordsIter<'a> {
     }
 }
 
+/// Per-chunk leaf value-count analysis derived from the metadata words.
+///
+/// `values_per_chunk` is the count shared by every non-last chunk (meaningful
+/// when `uniform`), and `last_chunk_values` is the final chunk's count.
+struct FlatValueCounts {
+    logs: Vec<u8>,
+    uniform: bool,
+    values_per_chunk: u64,
+    last_chunk_values: u64,
+}
+
+fn analyze_value_counts(words: &Words, items_in_page: u64) -> FlatValueCounts {
+    let num_chunks = words.len();
+    let logs = words.iter().map(|w| (w & 0x0F) as u8).collect::<Vec<_>>();
+    let mut counted = 0u64;
+    for &log in logs.iter().take(num_chunks.saturating_sub(1)) {
+        // Non-last chunks always encode a positive log value count.
+        debug_assert!(log > 0);
+        counted += 1u64 << log;
+    }
+    let last_chunk_values = items_in_page - counted;
+    if let Some(&last_log) = logs.last() {
+        debug_assert!(last_log == 0 || (1u64 << last_log) == last_chunk_values);
+    }
+    let uniform = num_chunks <= 1 || logs[..num_chunks - 1].iter().all(|&log| log == logs[0]);
+    // A single-chunk page has no "non-last" chunk to derive a stride from; use the
+    // page item count (min 1 so it stays a valid divisor in `find_chunk`).
+    let values_per_chunk = if num_chunks <= 1 {
+        items_in_page.max(1)
+    } else {
+        1u64 << logs[0]
+    };
+    FlatValueCounts {
+        logs,
+        uniform,
+        values_per_chunk,
+        last_chunk_values,
+    }
+}
+
+/// Iterator over per-chunk value counts for a non-uniform flat page.  Non-last
+/// chunks yield `1 << log`; the last yields the remaining items in the page.
+fn flat_value_counts_iter(logs: &[u8], items_in_page: u64) -> impl Iterator<Item = u64> + '_ {
+    let num_chunks = logs.len();
+    let mut counted = 0u64;
+    (0..num_chunks).map(move |i| {
+        let count = if i + 1 < num_chunks {
+            1u64 << logs[i]
+        } else {
+            items_in_page - counted
+        };
+        counted += count;
+        count
+    })
+}
+
+/// Builds the compact per-chunk index from the metadata words and, for nested
+/// pages, the raw repetition-index bytes.  The row axis is picked by page shape:
+/// `UniformFlat` when all non-last chunks share a value count (fixed-width /
+/// bitpacking), `Flat` for non-uniform flat pages (RLE / FSST), else `Nested`.
+fn build_chunk_index(
+    words: &Words,
+    items_in_page: u64,
+    base: u64,
+    data_buf_size: u64,
+    rep_index_bytes: Option<&[u8]>,
+    repetition_index_depth: u16,
+) -> MiniBlockChunkIndex {
+    let num_chunks = words.len();
+    // Each chunk stores `(divided_bytes + 1) * MINIBLOCK_ALIGNMENT` bytes, so the
+    // deltas are the chunk sizes and their grand total is the data buffer size.
+    let byte_starts = PrefixSums::from_deltas(
+        words
+            .iter()
+            .map(|word| ((word >> 4) as u64 + 1) * MINIBLOCK_ALIGNMENT as u64),
+        num_chunks,
+        data_buf_size,
+    );
+
+    // Nested pages track rows via the repetition index and keep leaf item counts
+    // separately; flat pages have row == value index, so value counts are rows.
+    let rows = if let Some(rep_index_data) = rep_index_bytes {
+        assert!(rep_index_data.len() % 8 == 0);
+        let stride = repetition_index_depth as usize + 1;
+        let (row_starts, has_trailer) = parse_nested_rep(rep_index_data, stride);
+        let value_counts = analyze_value_counts(words, items_in_page);
+        let item_counts = if value_counts.uniform {
+            ItemCounts::Uniform {
+                values_per_chunk: value_counts.values_per_chunk,
+                last_chunk_values: value_counts.last_chunk_values,
+            }
+        } else {
+            ItemCounts::PerChunkLog {
+                logs: value_counts.logs,
+                last_chunk_values: value_counts.last_chunk_values,
+            }
+        };
+        RowMapping::Nested {
+            row_starts,
+            has_trailer,
+            item_counts,
+        }
+    } else {
+        let value_counts = analyze_value_counts(words, items_in_page);
+        if value_counts.uniform {
+            RowMapping::UniformFlat {
+                values_per_chunk: value_counts.values_per_chunk,
+                last_chunk_values: value_counts.last_chunk_values,
+                num_chunks,
+            }
+        } else {
+            let value_starts = PrefixSums::from_deltas(
+                flat_value_counts_iter(&value_counts.logs, items_in_page),
+                num_chunks,
+                items_in_page,
+            );
+            RowMapping::Flat { value_starts }
+        }
+    };
+
+    MiniBlockChunkIndex::new(base, byte_starts, rows)
+}
+
 impl StructuralPageScheduler for MiniBlockScheduler {
     fn initialize<'a>(
         &'a mut self,
@@ -1845,7 +1862,8 @@ impl StructuralPageScheduler for MiniBlockScheduler {
         // we may also need to fetch the repetition index.  Here, we gather what buffers we
         // need.
         let (meta_buf_position, meta_buf_size) = self.buffer_offsets_and_sizes[0];
-        let value_buf_position = self.buffer_offsets_and_sizes[1].0;
+        let base = self.buffer_offsets_and_sizes[1].0;
+        let data_buf_size = self.buffer_offsets_and_sizes[1].1;
         let mut bufs_needed = 1;
         if self.dictionary.is_some() {
             bufs_needed += 1;
@@ -1874,65 +1892,31 @@ impl StructuralPageScheduler for MiniBlockScheduler {
             let dictionary_bytes = self.dictionary.as_ref().and_then(|_| buffers.next());
             let rep_index_bytes = buffers.next();
 
-            // Parse the metadata and build the chunk meta
             let words = Words::from_bytes(meta_bytes, self.has_large_chunk)?;
-            let mut chunk_meta = Vec::with_capacity(words.len());
-
-            let mut rows_counter = 0;
-            let mut offset_bytes = value_buf_position;
-            for (word_idx, word) in words.iter().enumerate() {
-                let log_num_values = word & 0x0F;
-                let divided_bytes = word >> 4;
-                let num_bytes = (divided_bytes as usize + 1) * MINIBLOCK_ALIGNMENT;
-                debug_assert!(num_bytes > 0);
-                let num_values = if word_idx < words.len() - 1 {
-                    debug_assert!(log_num_values > 0);
-                    1 << log_num_values
-                } else {
-                    debug_assert!(
-                        log_num_values == 0
-                            || (1 << log_num_values) == (self.items_in_page - rows_counter)
-                    );
-                    self.items_in_page - rows_counter
-                };
-                rows_counter += num_values;
-
-                chunk_meta.push(ChunkMeta {
-                    num_values,
-                    chunk_size_bytes: num_bytes as u64,
-                    offset_bytes,
-                });
-                offset_bytes += num_bytes as u64;
-            }
-
-            // Build the repetition index
-            let rep_index = if let Some(rep_index_data) = rep_index_bytes {
-                assert!(rep_index_data.len() % 8 == 0);
-                let stride = self.repetition_index_depth as usize + 1;
-                MiniBlockRepIndex::decode_from_bytes(&rep_index_data, stride)
-            } else {
-                MiniBlockRepIndex::default_from_chunks(&chunk_meta)
-            };
-
-            let mut page_meta = MiniBlockCacheableState {
-                chunk_meta,
-                rep_index,
-                dictionary: None,
-            };
+            let chunk_index = build_chunk_index(
+                &words,
+                self.items_in_page,
+                base,
+                data_buf_size,
+                rep_index_bytes.as_deref(),
+                self.repetition_index_depth,
+            );
 
             // decode dictionary
-            if let Some(ref mut dictionary) = self.dictionary {
+            let dictionary = if let Some(ref mut dictionary) = self.dictionary {
                 let dictionary_data = dictionary_bytes.unwrap();
-                page_meta.dictionary =
-                    Some(Arc::new(dictionary.dictionary_decompressor.decompress(
-                        LanceBuffer::from_bytes(
-                            dictionary_data,
-                            dictionary.dictionary_data_alignment,
-                        ),
-                        dictionary.num_dictionary_items,
-                    )?));
+                Some(Arc::new(dictionary.dictionary_decompressor.decompress(
+                    LanceBuffer::from_bytes(dictionary_data, dictionary.dictionary_data_alignment),
+                    dictionary.num_dictionary_items,
+                )?))
+            } else {
+                None
             };
-            let page_meta = Arc::new(page_meta);
+
+            let page_meta = Arc::new(MiniBlockCacheableState {
+                chunk_index,
+                dictionary,
+            });
             self.page_meta = Some(page_meta.clone());
             Ok(page_meta as Arc<dyn CachedPageData>)
         }
@@ -1958,7 +1942,7 @@ impl StructuralPageScheduler for MiniBlockScheduler {
         let page_meta = self.page_meta.as_ref().unwrap();
 
         let chunk_instructions =
-            ChunkInstructions::schedule_instructions(&page_meta.rep_index, ranges);
+            ChunkInstructions::schedule_instructions(&page_meta.chunk_index, ranges);
 
         debug_assert_eq!(
             num_rows,
@@ -3349,6 +3333,16 @@ impl StructuralPrimitiveFieldScheduler {
                 mini_block,
                 decompressors,
             )?),
+            Layout::SparseLayout(sparse_layout) => {
+                Box::new(sparse::SparseStructuralScheduler::try_new(
+                    &page_info.buffer_offsets_and_sizes,
+                    page_info.priority,
+                    page_info.num_rows,
+                    target_field.data_type(),
+                    sparse_layout,
+                    decompressors,
+                )?)
+            }
             Layout::FullZipLayout(full_zip) => {
                 let mut scheduler = FullZipScheduler::try_new(
                     &page_info.buffer_offsets_and_sizes,
@@ -3576,25 +3570,34 @@ impl StructuralCompositeDecodeArrayTask {
     fn restore_validity(
         array: Arc<dyn Array>,
         unraveler: &mut CompositeRepDefUnraveler,
-    ) -> Arc<dyn Array> {
-        let validity = unraveler.unravel_validity(array.len());
+    ) -> Result<Arc<dyn Array>> {
+        let validity = unraveler.unravel_validity(array.len())?;
         let Some(validity) = validity else {
-            return array;
+            return Ok(array);
         };
         if array.data_type() == &DataType::Null {
             // We unravel from a null array but we don't add the null buffer because arrow-rs doesn't like it
-            return array;
+            return Ok(array);
+        }
+        if validity.len() != array.len() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Structural validity has {} entries for an array with {} values",
+                    validity.len(),
+                    array.len()
+                )
+                .into(),
+            ));
         }
-        assert_eq!(validity.len(), array.len());
-        // SAFETY: We've should have already asserted the buffers are all valid, we are just
-        // adding null buffers to the array here
-        make_array(unsafe {
+        // SAFETY: The array buffers have already been validated and the null buffer length
+        // matches the array. We are only attaching the null buffer here.
+        Ok(make_array(unsafe {
             array
                 .to_data()
                 .into_builder()
                 .nulls(Some(validity))
                 .build_unchecked()
-        })
+        }))
     }
 }
 
@@ -3620,7 +3623,7 @@ impl StructuralDecodeArrayTask for StructuralCompositeDecodeArrayTask {
         let array = arrow_select::concat::concat(&array_refs)?;
         let mut repdef = CompositeRepDefUnraveler::new(unravelers);
 
-        let array = Self::restore_validity(array, &mut repdef);
+        let array = Self::restore_validity(array, &mut repdef)?;
 
         Ok(DecodedArray {
             array,
@@ -3787,18 +3790,27 @@ struct DictEncodingBudget {
     max_encoded_size: usize,
 }
 
-// A primitive page after optional structural splitting.
+enum PrimitivePageStructure {
+    Dense {
+        repdef: SerializedRepDefs,
+        single_row_miniblock_repdef_levels: Option<u64>,
+    },
+    Sparse {
+        plan: sparse::SparseStructuralPlan,
+        prepared_values: Option<sparse::writer::PreparedSparseValues>,
+    },
+}
+
+// A primitive page after structural encoding selection and optional dense splitting.
 struct PrimitivePageData {
     // Arrow leaf arrays that contain this page's visible values.
     arrays: Vec<ArrayRef>,
-    // Repetition / definition levels aligned to this page.
-    repdef: SerializedRepDefs,
+    // Structural representation aligned to this page.
+    structure: PrimitivePageStructure,
     // Top-level row number of the first row in this page.
     row_number: u64,
     // Number of top-level rows in this page.
     num_rows: u64,
-    // Present when one top-level row is too large for one miniblock rep/def chunk.
-    unsplittable_miniblock_levels: Option<u64>,
 }
 
 // Immutable encoder state shared by per-page encode tasks.
@@ -3833,6 +3845,19 @@ impl PrimitiveStructuralEncoder {
         field: Field,
         encoding_metadata: Arc<HashMap<String, String>>,
     ) -> Result<Self> {
+        let requests_sparse = encoding_metadata
+            .get(STRUCTURAL_ENCODING_META_KEY)
+            .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE));
+        if requests_sparse && options.version.resolve() < LanceFileVersion::V2_3 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Field '{}' requests sparse structural encoding, which requires Lance file format 2.3+; current version is {}",
+                    field.name,
+                    options.version.resolve()
+                )
+                .into(),
+            ));
+        }
         Ok(Self {
             accumulation_queue: AccumulationQueue::new(
                 options.cache_bytes_per_column,
@@ -4362,7 +4387,7 @@ impl PrimitiveStructuralEncoder {
             return Ok(None);
         }
         let mut validity = BooleanBufferBuilder::new(num_values);
-        unraveler.unravel_validity(&mut validity);
+        unraveler.unravel_validity(&mut validity)?;
         Ok(Some(validity.finish()))
     }
 
@@ -5292,33 +5317,37 @@ impl PrimitiveStructuralEncoder {
         Ok(sliced)
     }
 
-    fn split_structural_pages_for_miniblock_budget(
+    fn split_pages_for_miniblock_repdef_budget(
         arrays: Vec<ArrayRef>,
         repdef: SerializedRepDefs,
-        plan: StructuralPagePlan,
+        budget: MiniBlockRepDefBudget,
         row_number: u64,
         num_rows: u64,
     ) -> Result<Vec<PrimitivePageData>> {
-        if plan == StructuralPagePlan::Fits {
+        if budget == MiniBlockRepDefBudget::WithinBudget {
             return Ok(vec![PrimitivePageData {
                 arrays,
-                repdef,
+                structure: PrimitivePageStructure::Dense {
+                    repdef,
+                    single_row_miniblock_repdef_levels: None,
+                },
                 row_number,
                 num_rows,
-                unsplittable_miniblock_levels: None,
             }]);
         }
-        if let StructuralPagePlan::UnsplittableOverBudget(num_levels) = plan {
+        if let MiniBlockRepDefBudget::SingleRowOverBudget(num_levels) = budget {
             return Ok(vec![PrimitivePageData {
                 arrays,
-                repdef,
+                structure: PrimitivePageStructure::Dense {
+                    repdef,
+                    single_row_miniblock_repdef_levels: Some(num_levels),
+                },
                 row_number,
                 num_rows,
-                unsplittable_miniblock_levels: Some(num_levels),
             }]);
         }
 
-        let StructuralPagePlan::Split(splits) = plan else {
+        let MiniBlockRepDefBudget::RequiresPageSplit(splits) = budget else {
             unreachable!();
         };
 
@@ -5328,10 +5357,12 @@ impl PrimitiveStructuralEncoder {
             let repdef = Self::slice_repdef(&repdef, split.level_range);
             pages.push(PrimitivePageData {
                 arrays,
-                repdef,
+                structure: PrimitivePageStructure::Dense {
+                    repdef,
+                    single_row_miniblock_repdef_levels: None,
+                },
                 row_number: row_number + split.row_start,
                 num_rows: split.num_rows,
-                unsplittable_miniblock_levels: None,
             });
         }
         Ok(pages)
@@ -5350,13 +5381,47 @@ impl PrimitiveStructuralEncoder {
         } = ctx;
         let PrimitivePageData {
             arrays,
-            repdef,
+            structure,
             row_number,
             num_rows,
-            unsplittable_miniblock_levels,
         } = page;
         let num_values = arrays.iter().map(|arr| arr.len() as u64).sum();
 
+        let (repdef, single_row_miniblock_repdef_levels) = match structure {
+            PrimitivePageStructure::Dense {
+                repdef,
+                single_row_miniblock_repdef_levels,
+            } => (repdef, single_row_miniblock_repdef_levels),
+            PrimitivePageStructure::Sparse {
+                plan,
+                prepared_values,
+            } => {
+                log::debug!(
+                    "Encoding column {} with {} visible items ({} rows) using sparse layout",
+                    column_idx,
+                    num_values,
+                    num_rows
+                );
+                return sparse::writer::encode_page(
+                    column_idx,
+                    &field,
+                    compression_strategy.as_ref(),
+                    prepared_values.map_or_else(
+                        || {
+                            sparse::writer::SparseValueInput::Unprepared(DataBlock::from_arrays(
+                                &arrays, num_values,
+                            ))
+                        },
+                        sparse::writer::SparseValueInput::Prepared,
+                    ),
+                    plan,
+                    row_number,
+                    num_rows,
+                    support_large_chunk,
+                );
+            }
+        };
+
         if num_values == 0 {
             // This page contains only structural events, such as empty/null list rows.
             // The existing complex-null layout stores the rep/def stream without value buffers.
@@ -5436,7 +5501,7 @@ impl PrimitiveStructuralEncoder {
             );
         }
 
-        if let Some(num_levels) = unsplittable_miniblock_levels {
+        if let Some(num_levels) = single_row_miniblock_repdef_levels {
             let requested_encoding = encoding_metadata
                 .get(STRUCTURAL_ENCODING_META_KEY)
                 .map(|requested| requested.to_lowercase());
@@ -5663,19 +5728,82 @@ impl PrimitiveStructuralEncoder {
         let num_values = arrays.iter().map(|arr| arr.len() as u64).sum();
         let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity());
         let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty());
-        let (repdef, structural_plan) = RepDefBuilder::serialize_with_structural_plan(
-            repdefs,
-            miniblock::max_repdef_levels_per_chunk,
-            num_rows,
-            num_values,
-        )?;
-        let pages = Self::split_structural_pages_for_miniblock_budget(
-            arrays,
-            repdef,
-            structural_plan,
-            row_number,
-            num_rows,
-        )?;
+        let normalized = RepDefBuilder::normalize(repdefs);
+        let requested_encoding = self.encoding_metadata.get(STRUCTURAL_ENCODING_META_KEY);
+        let requests_sparse = requested_encoding
+            .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE));
+        let sparse_plan = requests_sparse
+            .then(|| sparse::writer::plan(&normalized, num_values))
+            .transpose()?;
+        let pages = if let Some(plan) = sparse_plan
+            && !sparse::writer::uses_constant_layout(&plan, &self.field)
+        {
+            vec![PrimitivePageData {
+                arrays,
+                structure: PrimitivePageStructure::Sparse {
+                    plan,
+                    prepared_values: None,
+                },
+                row_number,
+                num_rows,
+            }]
+        } else {
+            let (repdef, miniblock_repdef_budget) = normalized
+                .serialize_with_miniblock_repdef_budget(
+                    miniblock::max_repdef_levels_per_chunk,
+                    num_rows,
+                    num_values,
+                )?;
+            let automatic_sparse = layout::select_automatic_sparse(
+                self.version.resolve(),
+                requested_encoding.map(String::as_str),
+                &miniblock_repdef_budget,
+                || {
+                    let data = DataBlock::from_arrays(&arrays, num_values);
+                    if !sparse::writer::supports_value_block(&data) {
+                        return Ok(None);
+                    }
+                    let prepared_values = match sparse::writer::prepare_values(
+                        &self.field,
+                        self.compression_strategy.as_ref(),
+                        data,
+                        self.support_large_chunk,
+                    ) {
+                        Ok(prepared_values) => prepared_values,
+                        Err(error) => {
+                            debug!(
+                                "Keeping column {} on its dense structural path because sparse value preparation is unavailable: {}",
+                                self.column_index, error
+                            );
+                            return Ok(None);
+                        }
+                    };
+                    let plan = sparse::writer::plan(&normalized, num_values)?;
+                    if sparse::writer::uses_constant_layout(&plan, &self.field) {
+                        return Ok(None);
+                    }
+                    Ok(Some((plan, prepared_values)))
+                },
+            )?;
+            match automatic_sparse {
+                Some((plan, prepared_values)) => vec![PrimitivePageData {
+                    arrays,
+                    structure: PrimitivePageStructure::Sparse {
+                        plan,
+                        prepared_values: Some(prepared_values),
+                    },
+                    row_number,
+                    num_rows,
+                }],
+                None => Self::split_pages_for_miniblock_repdef_budget(
+                    arrays,
+                    repdef,
+                    miniblock_repdef_budget,
+                    row_number,
+                    num_rows,
+                )?,
+            }
+        };
 
         let mut tasks = Vec::with_capacity(pages.len());
         let ctx = PrimitiveEncodeContext {
@@ -5793,8 +5921,8 @@ mod tests {
     use super::{
         ChunkInstructions, DataBlock, DecodeMiniBlockTask, FixedPerValueDecompressor,
         FixedWidthDataBlock, FullZipCacheableState, FullZipDecodeDetails, FullZipReadSource,
-        FullZipRepIndexDetails, FullZipScheduler, MiniBlockChunk, MiniBlockCompressed,
-        MiniBlockRepIndex, PerValueDecompressor, PreambleAction, StructuralPageScheduler,
+        FullZipRepIndexDetails, FullZipScheduler, MiniBlockChunk, MiniBlockChunkIndex,
+        MiniBlockCompressed, PerValueDecompressor, PreambleAction, StructuralPageScheduler,
         VariableFullZipDecoder,
     };
     use crate::buffer::LanceBuffer;
@@ -6289,11 +6417,10 @@ mod tests {
         // Convert repetition index to bytes for testing
         let rep_data: Vec<u64> = vec![5, 2, 3, 0, 4, 7, 2, 0];
         let rep_bytes: Vec<u8> = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect();
-        let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2);
+        let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2);
 
         let check = |user_ranges, expected_instructions| {
-            let instructions =
-                ChunkInstructions::schedule_instructions(&repetition_index, user_ranges);
+            let instructions = ChunkInstructions::schedule_instructions(&chunk_index, user_ranges);
             assert_eq!(instructions, expected_instructions);
         };
 
@@ -6445,11 +6572,11 @@ mod tests {
         // Convert repetition index to bytes for testing
         let rep_data: Vec<u64> = vec![5, 2, 3, 0, 4, 7, 2, 0];
         let rep_bytes: Vec<u8> = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect();
-        let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2);
+        let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2);
         let user_ranges = vec![1..7, 10..14];
 
         // First, schedule the ranges
-        let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges);
+        let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges);
 
         let mut to_drain = VecDeque::from(scheduled.clone());
 
@@ -6530,11 +6657,11 @@ mod tests {
         // Regression case.  Need a chunk with preamble, rows, and trailer (the middle chunk here)
         let rep_data: Vec<u64> = vec![5, 2, 3, 3, 20, 0];
         let rep_bytes: Vec<u8> = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect();
-        let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2);
+        let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2);
         let user_ranges = vec![0..28];
 
         // First, schedule the ranges
-        let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges);
+        let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges);
 
         let mut to_drain = VecDeque::from(scheduled.clone());
 
@@ -6594,6 +6721,181 @@ mod tests {
         assert_eq!(skip_in_chunk, 0);
     }
 
+    use super::chunk_index::{PrefixSums, RowMapping};
+    use super::{MINIBLOCK_ALIGNMENT, Words, build_chunk_index};
+    use bytes::Bytes;
+    use lance_core::cache::{Context, DeepSizeOf};
+    use rstest::rstest;
+
+    /// Builds a `Words` metadata buffer (u16 words) from `(log_num_values, num_bytes)`
+    /// pairs, returning the words and the total data-buffer size.
+    fn words_from(entries: &[(u32, u32)]) -> (Words, u64) {
+        let mut raw = Vec::with_capacity(entries.len() * 2);
+        let mut total = 0u64;
+        for &(log, num_bytes) in entries {
+            assert!(num_bytes > 0 && num_bytes % MINIBLOCK_ALIGNMENT as u32 == 0);
+            let divided = num_bytes / MINIBLOCK_ALIGNMENT as u32 - 1;
+            let word = (divided << 4) | log;
+            assert!(word <= u16::MAX as u32, "test word {word} exceeds u16");
+            raw.extend_from_slice(&(word as u16).to_le_bytes());
+            total += num_bytes as u64;
+        }
+        (Words::from_bytes(Bytes::from(raw), false).unwrap(), total)
+    }
+
+    fn rep_bytes_from(values: &[u64]) -> Vec<u8> {
+        values.iter().flat_map(|v| v.to_le_bytes()).collect()
+    }
+
+    #[rstest]
+    // Two full chunks of 8 values (log 3) plus a partial last chunk; byte sizes vary
+    // independently of value counts.
+    #[case::uniform_partial_last(&[(3, 16), (3, 24), (0, 8)], 19, "uniform_flat", 8, 3)]
+    // Single chunk covers the whole page.
+    #[case::single_chunk(&[(0, 24)], 5, "uniform_flat", 5, 5)]
+    // Last chunk is also full (exact multiple).
+    #[case::exact_multiple(&[(3, 16), (3, 16)], 16, "uniform_flat", 8, 8)]
+    // Non-last chunks differ in size, so this is a non-uniform flat page.
+    #[case::non_uniform(&[(4, 16), (2, 16), (0, 8)], 21, "flat", 16, 1)]
+    fn test_flat_detection(
+        #[case] entries: &[(u32, u32)],
+        #[case] items_in_page: u64,
+        #[case] expected_kind: &str,
+        #[case] expected_first_items: u64,
+        #[case] expected_last_items: u64,
+    ) {
+        let base = 100u64;
+        let (words, data_buf_size) = words_from(entries);
+        let index = build_chunk_index(&words, items_in_page, base, data_buf_size, None, 0);
+
+        assert_eq!(index.row_mapping_debug(), expected_kind);
+        assert_eq!(index.num_chunks(), entries.len());
+        assert_eq!(index.items_in_chunk(0), expected_first_items);
+        assert_eq!(index.items_in_chunk(entries.len() - 1), expected_last_items);
+
+        // Byte ranges are absolute, contiguous, and exactly cover the data buffer.
+        let mut expected_start = base;
+        for (i, &(_, num_bytes)) in entries.iter().enumerate() {
+            let range = index.byte_range(i);
+            assert_eq!(range.start, expected_start);
+            assert_eq!(range.end - range.start, num_bytes as u64);
+            expected_start = range.end;
+        }
+        assert_eq!(expected_start, base + data_buf_size);
+
+        // For flat pages rows == items, so the per-chunk items sum to the page total.
+        let total_items: u64 = (0..index.num_chunks())
+            .map(|i| index.items_in_chunk(i))
+            .sum();
+        assert_eq!(total_items, items_in_page);
+    }
+
+    #[test]
+    fn test_nested_detection_and_axes() {
+        // Repetition index (stride 2): three chunks holding 5, 4, 3 rows, no trailers.
+        let rep = rep_bytes_from(&[5, 0, 4, 0, 3, 0]);
+
+        // Uniform leaf chunking: value counts 4, 4, 2.
+        let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (0, 8)]);
+        let index = build_chunk_index(&words, 10, 0, data_buf_size, Some(&rep), 1);
+        assert_eq!(index.row_mapping_debug(), "nested");
+        assert_eq!(index.num_chunks(), 3);
+        // Rows come from the repetition index, not the value counts.
+        assert_eq!(index.first_row(0), 0);
+        assert_eq!(index.rows_in_chunk(0), 5);
+        assert_eq!(index.first_row(1), 5);
+        assert_eq!(index.rows_in_chunk(1), 4);
+        assert_eq!(index.first_row(2), 9);
+        assert_eq!(index.rows_in_chunk(2), 3);
+        // Items come from the value words.
+        assert_eq!(index.items_in_chunk(0), 4);
+        assert_eq!(index.items_in_chunk(1), 4);
+        assert_eq!(index.items_in_chunk(2), 2);
+
+        // Non-uniform leaf chunking: value counts 8, 2, 5.
+        let (words_nu, dbs_nu) = words_from(&[(3, 8), (1, 8), (0, 8)]);
+        let index_nu = build_chunk_index(&words_nu, 15, 0, dbs_nu, Some(&rep), 1);
+        assert_eq!(index_nu.row_mapping_debug(), "nested");
+        assert_eq!(index_nu.items_in_chunk(0), 8);
+        assert_eq!(index_nu.items_in_chunk(1), 2);
+        assert_eq!(index_nu.items_in_chunk(2), 5);
+        // The row axis is unchanged by the leaf chunking.
+        assert_eq!(index_nu.rows_in_chunk(0), 5);
+    }
+
+    #[test]
+    fn test_uniform_flat_matches_prefix_sum_flat() {
+        // Distribution: 4 chunks of 4 values, last chunk 3 (15 items total).
+        let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (2, 8), (0, 8)]);
+        let uniform = build_chunk_index(&words, 15, 0, data_buf_size, None, 0);
+        assert_eq!(uniform.row_mapping_debug(), "uniform_flat");
+
+        // The same distribution expressed as a non-uniform Flat prefix-sum index.
+        let byte_starts = PrefixSums::from_deltas([8u64, 8, 8, 8].into_iter(), 4, 32);
+        let value_starts = PrefixSums::from_deltas([4u64, 4, 4, 3].into_iter(), 4, 15);
+        let flat = MiniBlockChunkIndex::new(0, byte_starts, RowMapping::Flat { value_starts });
+        assert_eq!(flat.row_mapping_debug(), "flat");
+
+        // Lookup parity: identical byte ranges and item counts.
+        for i in 0..4 {
+            assert_eq!(uniform.byte_range(i), flat.byte_range(i));
+            assert_eq!(uniform.items_in_chunk(i), flat.items_in_chunk(i));
+        }
+
+        // Scheduler parity across scan / single-row / partial / scattered multi-range.
+        let range_sets: Vec<Vec<std::ops::Range<u64>>> = vec![
+            vec![0..15],
+            vec![0..1],
+            vec![7..8],
+            vec![14..15],
+            vec![3..10],
+            vec![0..2, 5..6, 12..15],
+        ];
+        for ranges in &range_sets {
+            let from_uniform = ChunkInstructions::schedule_instructions(&uniform, ranges);
+            let from_flat = ChunkInstructions::schedule_instructions(&flat, ranges);
+            assert_eq!(from_uniform, from_flat, "mismatch for ranges {ranges:?}");
+        }
+
+        // A full scan yields one Absent, no-trailer instruction per chunk.
+        let full = ChunkInstructions::schedule_instructions(&uniform, &[0..15]);
+        assert_eq!(full.len(), 4);
+        for (i, inst) in full.iter().enumerate() {
+            assert_eq!(inst.chunk_idx, i);
+            assert_eq!(inst.preamble, PreambleAction::Absent);
+            assert_eq!(inst.rows_to_skip, 0);
+            assert!(!inst.take_trailer);
+        }
+        assert_eq!(full.iter().map(|i| i.rows_to_take).sum::<u64>(), 15);
+    }
+
+    #[test]
+    fn test_deep_size_per_variant_below_legacy() {
+        // The previous representation cached 48 bytes per chunk (24 for ChunkMeta plus
+        // 24 for a rep-index block); every variant's heap must be well below that.
+        const LEGACY_PER_CHUNK: usize = 48;
+        let num_chunks = 3;
+        let heap = |index: &MiniBlockChunkIndex| index.deep_size_of_children(&mut Context::new());
+
+        let (uniform_words, uniform_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]);
+        let uniform = build_chunk_index(&uniform_words, 10, 0, uniform_dbs, None, 0);
+        assert_eq!(uniform.row_mapping_debug(), "uniform_flat");
+        assert!(heap(&uniform) < LEGACY_PER_CHUNK * num_chunks);
+
+        let (flat_words, flat_dbs) = words_from(&[(3, 8), (1, 8), (0, 8)]);
+        let flat = build_chunk_index(&flat_words, 11, 0, flat_dbs, None, 0);
+        assert_eq!(flat.row_mapping_debug(), "flat");
+        assert!(heap(&flat) < LEGACY_PER_CHUNK * num_chunks);
+        // Flat carries a value-starts array that UniformFlat derives arithmetically.
+        assert!(heap(&flat) > heap(&uniform));
+
+        let rep = rep_bytes_from(&[4, 0, 3, 0, 3, 0]);
+        let (nested_words, nested_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]);
+        let nested = build_chunk_index(&nested_words, 10, 0, nested_dbs, Some(&rep), 1);
+        assert_eq!(nested.row_mapping_debug(), "nested");
+        assert!(heap(&nested) < LEGACY_PER_CHUNK * num_chunks);
+    }
+
     #[tokio::test]
     async fn test_fullzip_initialize_is_lazy() {
         use futures::{FutureExt, future::BoxFuture};
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
index eed3e584b7e..52a7039b2b6 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
@@ -258,7 +258,9 @@ impl BlobPageScheduler {
             let bytes = read_fut.await?;
             let mut bytes_iter = bytes.into_iter();
             for blob in loaded_blobs.iter_mut() {
-                if blob.def == 0 {
+                // Empty values have def == 0 too but scheduled no read; their
+                // bytes were set at scheduling time.
+                if blob.def == 0 && blob.bytes.is_none() {
                     blob.set_bytes(bytes_iter.next().expect_ok()?);
                 }
             }
@@ -364,7 +366,17 @@ impl StructuralPageScheduler for BlobPageScheduler {
                 if size == 0 {
                     let rep = (position & 0xFFFF) as u16;
                     let def = ((position >> 16) & 0xFFFF) as u16;
-                    loaded_blobs.push(LoadedBlob::new(rep, def));
+                    let mut blob = LoadedBlob::new(rep, def);
+                    if def == 0 {
+                        // A size-0 descriptor with definition level 0 is a
+                        // valid, empty value (nulls carry their non-zero
+                        // packed rep/def levels in `position`). No read is
+                        // scheduled for it, so it gets its zero-length bytes
+                        // here rather than consuming another blob's read
+                        // result in the load task.
+                        blob.set_bytes(Bytes::new());
+                    }
+                    loaded_blobs.push(blob);
                 } else {
                     loaded_blobs.push(LoadedBlob::new(0, 0));
                     ranges_to_read.push(position..(position + size));
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs
new file mode 100644
index 00000000000..19fb0fdb041
--- /dev/null
+++ b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs
@@ -0,0 +1,536 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Compact per-page chunk index for the mini-block structural encoding.
+//!
+//! The chunk index is stored on disk in an extremely compressed form that
+//! requires a lot of CPU to work with.  However, extracting it out to its full
+//! width can be RAM-intensive.  As a compromise we extract into a prefix-sum
+//! array that we fit into `u32` if possible and we avoid storing per-block row
+//! counts when those are redundant.  The scheduler looks chunks up by index
+//! (byte range, leaf value count) and by row (which chunk holds a row).
+//!
+//! ```text
+//! MiniBlockChunkIndex
+//! |- base: u64                      absolute file position of the value buffer
+//! |- byte_starts: PrefixSums        cumulative chunk byte sizes (all pages)
+//! `- rows: RowMapping
+//!    |- UniformFlat { .. }          flat page, uniform leaf chunking (arithmetic)
+//!    |- Flat { value_starts }       flat page, non-uniform leaf chunking
+//!    `- Nested { row_starts, .. }   repetition present; rows tracked as prefix sums
+//! ```
+
+use std::ops::Range;
+
+use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder};
+use lance_core::cache::{Context, DeepSizeOf};
+
+/// Cumulative (prefix-sum) array of length `num_chunks + 1` (entry `0` is `0`,
+/// the last entry is the grand total).  Stored as `u32` when the total fits,
+/// else `u64`.
+#[derive(Debug, DeepSizeOf)]
+pub enum PrefixSums {
+    U32(Vec<u32>),
+    U64(Vec<u64>),
+}
+
+impl PrefixSums {
+    /// Builds the cumulative array from per-chunk `deltas`.  `total` selects the
+    /// storage width (callers must pass the true sum); `num_chunks` only pre-sizes.
+    pub fn from_deltas(deltas: impl Iterator<Item = u64>, num_chunks: usize, total: u64) -> Self {
+        if total <= u32::MAX as u64 {
+            let mut values = Vec::with_capacity(num_chunks + 1);
+            let mut acc = 0u32;
+            values.push(0);
+            for delta in deltas {
+                acc += delta as u32;
+                values.push(acc);
+            }
+            debug_assert_eq!(values.len(), num_chunks + 1);
+            debug_assert_eq!(acc as u64, total);
+            Self::U32(values)
+        } else {
+            let mut values = Vec::with_capacity(num_chunks + 1);
+            let mut acc = 0u64;
+            values.push(0);
+            for delta in deltas {
+                acc += delta;
+                values.push(acc);
+            }
+            debug_assert_eq!(values.len(), num_chunks + 1);
+            debug_assert_eq!(acc, total);
+            Self::U64(values)
+        }
+    }
+
+    /// Builds a `PrefixSums` from an already-cumulative array (`[0, .., total]`),
+    /// narrowing to `u32` when the total fits.  Avoids the deltas buffer
+    /// [`Self::from_deltas`] would need.
+    fn from_prefix(prefix: Vec<u64>) -> Self {
+        debug_assert!(!prefix.is_empty());
+        debug_assert_eq!(prefix[0], 0);
+        let total = prefix.last().copied().unwrap_or(0);
+        if total <= u32::MAX as u64 {
+            Self::U32(prefix.into_iter().map(|v| v as u32).collect())
+        } else {
+            Self::U64(prefix)
+        }
+    }
+
+    /// Cumulative value at position `i` (i.e. the start of chunk `i`).
+    pub fn get(&self, i: usize) -> u64 {
+        match self {
+            Self::U32(values) => values[i] as u64,
+            Self::U64(values) => values[i],
+        }
+    }
+
+    /// Start and end of chunk `i` (positions `i`, `i + 1`) behind one width
+    /// match -- halves the branching of two `get` calls on the hot per-chunk path.
+    pub fn get_pair(&self, i: usize) -> (u64, u64) {
+        match self {
+            Self::U32(values) => (values[i] as u64, values[i + 1] as u64),
+            Self::U64(values) => (values[i], values[i + 1]),
+        }
+    }
+
+    /// Number of chunks (array length minus the trailing total).
+    pub fn num_chunks(&self) -> usize {
+        match self {
+            Self::U32(values) => values.len() - 1,
+            Self::U64(values) => values.len() - 1,
+        }
+    }
+
+    /// Size of chunk `i` (the delta between consecutive cumulative values).
+    pub fn delta(&self, i: usize) -> u64 {
+        let (start, end) = self.get_pair(i);
+        end - start
+    }
+
+    /// Index of the chunk whose half-open span `[get(i), get(i+1))` contains
+    /// `value`.  On an exact hit against a chunk start, returns the *first* chunk
+    /// with that start (chunks can share a start row).
+    pub fn find(&self, value: u64) -> usize {
+        // Match the width once, then binary-search only the starts (not the
+        // trailing total).  `partition_point` already yields the first of any
+        // duplicated starts; the `idx - 1` fallback is safe since `get(0) == 0`.
+        match self {
+            Self::U32(values) => {
+                let starts = &values[..values.len() - 1];
+                let idx = starts.partition_point(|&start| (start as u64) < value);
+                if idx < starts.len() && starts[idx] as u64 == value {
+                    idx
+                } else {
+                    idx - 1
+                }
+            }
+            Self::U64(values) => {
+                let starts = &values[..values.len() - 1];
+                let idx = starts.partition_point(|&start| start < value);
+                if idx < starts.len() && starts[idx] == value {
+                    idx
+                } else {
+                    idx - 1
+                }
+            }
+        }
+    }
+}
+
+/// Leaf value counts per chunk, needed to decode.  Tracked only for nested
+/// pages; flat pages read items off the row mapping (rows == items).
+#[derive(Debug, DeepSizeOf)]
+pub enum ItemCounts {
+    /// Every non-last chunk holds the same number of values.
+    Uniform {
+        values_per_chunk: u64,
+        last_chunk_values: u64,
+    },
+    /// `log2` of each chunk's value count, stored as one byte per chunk rather
+    /// than the full count because this index stays cached in RAM; the last
+    /// chunk is handled via `last_chunk_values`.
+    PerChunkLog {
+        logs: Vec<u8>,
+        last_chunk_values: u64,
+    },
+}
+
+impl ItemCounts {
+    fn get(&self, i: usize, num_chunks: usize) -> u64 {
+        match self {
+            Self::Uniform {
+                values_per_chunk,
+                last_chunk_values,
+            } => {
+                if i == num_chunks - 1 {
+                    *last_chunk_values
+                } else {
+                    *values_per_chunk
+                }
+            }
+            Self::PerChunkLog {
+                logs,
+                last_chunk_values,
+            } => {
+                if i == num_chunks - 1 {
+                    *last_chunk_values
+                } else {
+                    1u64 << logs[i]
+                }
+            }
+        }
+    }
+}
+
+/// How row ranges map onto chunks.
+///
+/// Flat pages have row == value index and no preamble/trailer.  Nested pages
+/// track rows separately from leaf items and store a trailer bit per chunk; a
+/// chunk's preamble is the previous chunk's trailer.
+#[derive(Debug)]
+pub enum RowMapping {
+    /// Flat page whose non-last chunks all hold `values_per_chunk` values, so
+    /// row->chunk is pure arithmetic.
+    UniformFlat {
+        values_per_chunk: u64,
+        last_chunk_values: u64,
+        num_chunks: usize,
+    },
+    /// Flat page with non-uniform chunk sizes; `value_starts` are the cumulative
+    /// value counts (final entry == number of items in the page).
+    Flat { value_starts: PrefixSums },
+    /// Nested page.  `row_starts` are cumulative row counts (final entry ==
+    /// number of rows in the page); `has_trailer[i]` is set when chunk `i` ends
+    /// with a partial list.
+    Nested {
+        row_starts: PrefixSums,
+        has_trailer: BooleanBuffer,
+        item_counts: ItemCounts,
+    },
+}
+
+impl DeepSizeOf for RowMapping {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        match self {
+            Self::UniformFlat { .. } => 0,
+            Self::Flat { value_starts } => value_starts.deep_size_of_children(context),
+            Self::Nested {
+                row_starts,
+                has_trailer,
+                item_counts,
+            } => {
+                row_starts.deep_size_of_children(context)
+                    + has_trailer.len().div_ceil(8)
+                    + item_counts.deep_size_of_children(context)
+            }
+        }
+    }
+}
+
+/// Compact per-page chunk index that avoids fully materializing the repetition
+/// index into u64's to save RAM.  See the module docs for the layout.
+#[derive(Debug)]
+pub struct MiniBlockChunkIndex {
+    base: u64,
+    byte_starts: PrefixSums,
+    rows: RowMapping,
+}
+
+impl MiniBlockChunkIndex {
+    pub fn new(base: u64, byte_starts: PrefixSums, rows: RowMapping) -> Self {
+        Self {
+            base,
+            byte_starts,
+            rows,
+        }
+    }
+
+    /// Number of chunks in the page.
+    pub fn num_chunks(&self) -> usize {
+        self.byte_starts.num_chunks()
+    }
+
+    /// Absolute byte range of chunk `i` within the file.
+    pub fn byte_range(&self, i: usize) -> Range<u64> {
+        let (start, end) = self.byte_starts.get_pair(i);
+        (self.base + start)..(self.base + end)
+    }
+
+    /// Number of leaf values in chunk `i` (passed to the value decompressor).
+    pub fn items_in_chunk(&self, i: usize) -> u64 {
+        let num_chunks = self.num_chunks();
+        match &self.rows {
+            RowMapping::UniformFlat {
+                values_per_chunk,
+                last_chunk_values,
+                ..
+            } => {
+                if i == num_chunks - 1 {
+                    *last_chunk_values
+                } else {
+                    *values_per_chunk
+                }
+            }
+            RowMapping::Flat { value_starts } => value_starts.delta(i),
+            RowMapping::Nested { item_counts, .. } => item_counts.get(i, num_chunks),
+        }
+    }
+
+    /// Index of the chunk that contains `row`.
+    pub fn find_chunk(&self, row: u64) -> usize {
+        match &self.rows {
+            RowMapping::UniformFlat {
+                values_per_chunk,
+                num_chunks,
+                ..
+            } => ((row / values_per_chunk) as usize).min(num_chunks - 1),
+            RowMapping::Flat { value_starts } => value_starts.find(row),
+            RowMapping::Nested { row_starts, .. } => row_starts.find(row),
+        }
+    }
+
+    /// First row (relative to the page) that begins in chunk `i`.
+    pub fn first_row(&self, i: usize) -> u64 {
+        match &self.rows {
+            RowMapping::UniformFlat {
+                values_per_chunk, ..
+            } => i as u64 * values_per_chunk,
+            RowMapping::Flat { value_starts } => value_starts.get(i),
+            RowMapping::Nested { row_starts, .. } => row_starts.get(i),
+        }
+    }
+
+    /// Number of rows that start in chunk `i`, including a trailer but not a
+    /// preamble (the previous `starts_including_trailer`).
+    pub fn rows_in_chunk(&self, i: usize) -> u64 {
+        let num_chunks = self.num_chunks();
+        match &self.rows {
+            RowMapping::UniformFlat {
+                values_per_chunk,
+                last_chunk_values,
+                ..
+            } => {
+                if i == num_chunks - 1 {
+                    *last_chunk_values
+                } else {
+                    *values_per_chunk
+                }
+            }
+            RowMapping::Flat { value_starts } => value_starts.delta(i),
+            RowMapping::Nested { row_starts, .. } => row_starts.delta(i),
+        }
+    }
+
+    /// Whether chunk `i` begins with a preamble (a continuation of the previous
+    /// chunk's list).  Always false for flat pages; for nested pages this is the
+    /// previous chunk's trailer.
+    pub fn has_preamble(&self, i: usize) -> bool {
+        match &self.rows {
+            RowMapping::Nested { has_trailer, .. } => i > 0 && has_trailer.value(i - 1),
+            _ => false,
+        }
+    }
+
+    /// Whether chunk `i` ends with a trailer (a partial list continued in the
+    /// next chunk).  Always false for flat pages.
+    pub fn has_trailer(&self, i: usize) -> bool {
+        match &self.rows {
+            RowMapping::Nested { has_trailer, .. } => has_trailer.value(i),
+            _ => false,
+        }
+    }
+
+    /// Name of the active row-mapping variant, used to assert detection in tests.
+    #[cfg(test)]
+    pub fn row_mapping_debug(&self) -> &'static str {
+        match &self.rows {
+            RowMapping::UniformFlat { .. } => "uniform_flat",
+            RowMapping::Flat { .. } => "flat",
+            RowMapping::Nested { .. } => "nested",
+        }
+    }
+
+    /// Builds a nested index from raw repetition-index bytes, using placeholder
+    /// byte offsets and item counts.  Only the row axis is populated, which is
+    /// all the scheduler exercises.
+    #[cfg(test)]
+    pub fn new_nested_for_test(rep_bytes: &[u8], stride: usize) -> Self {
+        let (row_starts, has_trailer) = parse_nested_rep(rep_bytes, stride);
+        let num_chunks = row_starts.num_chunks();
+        let byte_starts = PrefixSums::from_deltas(
+            std::iter::repeat_n(8u64, num_chunks),
+            num_chunks,
+            8 * num_chunks as u64,
+        );
+        Self {
+            base: 0,
+            byte_starts,
+            rows: RowMapping::Nested {
+                row_starts,
+                has_trailer,
+                item_counts: ItemCounts::Uniform {
+                    values_per_chunk: 1,
+                    last_chunk_values: 1,
+                },
+            },
+        }
+    }
+}
+
+impl DeepSizeOf for MiniBlockChunkIndex {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.byte_starts.deep_size_of_children(context) + self.rows.deep_size_of_children(context)
+    }
+}
+
+/// Parses a mini-block repetition index into the compact nested row mapping.
+///
+/// Bytes are `u64`s in groups of `stride`; the first two are `ends` (lists
+/// finishing in the chunk) and `partial` (leftover items).  Only cumulative row
+/// starts and a trailer bit are kept: `has_preamble[i] = has_trailer[i-1]` and
+/// `starts_including_trailer = ends + has_trailer - has_preamble`.
+pub fn parse_nested_rep(rep_bytes: &[u8], stride: usize) -> (PrefixSums, BooleanBuffer) {
+    // Read the two `u64`s per group straight from the little-endian bytes rather
+    // than copying the buffer to reinterpret it.  The caller guarantees
+    // `rep_bytes.len() % 8 == 0`, so the 8-byte windows stay in bounds.
+    const WORD: usize = std::mem::size_of::<u64>();
+    let read_word = |word_idx: usize| -> u64 {
+        let byte = word_idx * WORD;
+        u64::from_le_bytes(rep_bytes[byte..byte + WORD].try_into().unwrap())
+    };
+    let num_chunks = (rep_bytes.len() / WORD) / stride;
+
+    let mut has_trailer_builder = BooleanBufferBuilder::new(num_chunks);
+    // Accumulate the cumulative row starts in a single pass (entry 0 is 0, the
+    // trailing entry is the total) so there is no separate deltas buffer.
+    let mut row_starts = Vec::with_capacity(num_chunks + 1);
+    row_starts.push(0u64);
+    let mut acc = 0u64;
+    let mut chunk_has_preamble = false;
+
+    for i in 0..num_chunks {
+        let base_idx = i * stride;
+        let ends = read_word(base_idx);
+        let partial = read_word(base_idx + 1);
+
+        let has_trailer = partial > 0;
+        let starts_including_trailer = ends + (has_trailer as u64) - (chunk_has_preamble as u64);
+
+        has_trailer_builder.append(has_trailer);
+        acc += starts_including_trailer;
+        row_starts.push(acc);
+
+        chunk_has_preamble = has_trailer;
+    }
+
+    (
+        PrefixSums::from_prefix(row_starts),
+        has_trailer_builder.finish(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::buffer::LanceBuffer;
+
+    /// Reference decode: the previous per-block repetition index, used as an
+    /// oracle for the compact `parse_nested_rep`.
+    struct RefBlock {
+        first_row: u64,
+        starts_including_trailer: u64,
+        has_preamble: bool,
+        has_trailer: bool,
+    }
+
+    fn reference_decode(rep_bytes: &[u8], stride: usize) -> Vec<RefBlock> {
+        let buffer = LanceBuffer::from(rep_bytes.to_vec());
+        let u64_slice = buffer.borrow_to_typed_slice::<u64>();
+        let n = u64_slice.len() / stride;
+        let mut blocks = Vec::with_capacity(n);
+        let mut chunk_has_preamble = false;
+        let mut offset = 0u64;
+        for i in 0..n {
+            let base_idx = i * stride;
+            let ends = u64_slice[base_idx];
+            let partial = u64_slice[base_idx + 1];
+            let has_trailer = partial > 0;
+            let starts_including_trailer =
+                ends + (has_trailer as u64) - (chunk_has_preamble as u64);
+            blocks.push(RefBlock {
+                first_row: offset,
+                starts_including_trailer,
+                has_preamble: chunk_has_preamble,
+                has_trailer,
+            });
+            chunk_has_preamble = has_trailer;
+            offset += starts_including_trailer;
+        }
+        blocks
+    }
+
+    fn rep_bytes(values: &[u64]) -> Vec<u8> {
+        values.iter().flat_map(|v| v.to_le_bytes()).collect()
+    }
+
+    #[test]
+    fn test_prefix_sums_u32() {
+        let sums = PrefixSums::from_deltas([2u64, 3, 5].into_iter(), 3, 10);
+        assert!(matches!(sums, PrefixSums::U32(_)));
+        assert_eq!(sums.num_chunks(), 3);
+        assert_eq!(sums.get(0), 0);
+        assert_eq!(sums.get(1), 2);
+        assert_eq!(sums.get(3), 10);
+        assert_eq!(sums.delta(1), 3);
+    }
+
+    #[test]
+    fn test_prefix_sums_u64_selected_by_total() {
+        let big = u32::MAX as u64 + 1;
+        let sums = PrefixSums::from_deltas([big].into_iter(), 1, big);
+        assert!(matches!(sums, PrefixSums::U64(_)));
+        assert_eq!(sums.get(1), big);
+    }
+
+    #[test]
+    fn test_prefix_sums_find() {
+        // Chunk starts: 0, 5, 5, 12 (a zero-width chunk creates a duplicate start)
+        let sums = PrefixSums::from_deltas([5u64, 0, 7].into_iter(), 3, 12);
+        // Inside the first chunk
+        assert_eq!(sums.find(3), 0);
+        // Exact match on a duplicated start returns the first such chunk
+        assert_eq!(sums.find(5), 1);
+        // Inside the last chunk
+        assert_eq!(sums.find(11), 2);
+        // Start of the first chunk
+        assert_eq!(sums.find(0), 0);
+    }
+
+    #[test]
+    fn test_parse_nested_rep_matches_reference() {
+        let cases: Vec<Vec<u64>> = vec![
+            vec![5, 2, 3, 0, 4, 7, 2, 0],
+            vec![5, 2, 3, 3, 20, 0],
+            vec![0, 5, 0, 3, 10, 0],
+            vec![1, 0],
+        ];
+        for values in cases {
+            let bytes = rep_bytes(&values);
+            let reference = reference_decode(&bytes, 2);
+            let (row_starts, has_trailer) = parse_nested_rep(&bytes, 2);
+            assert_eq!(row_starts.num_chunks(), reference.len());
+            for (i, block) in reference.iter().enumerate() {
+                assert_eq!(row_starts.get(i), block.first_row, "first_row[{i}]");
+                assert_eq!(
+                    row_starts.delta(i),
+                    block.starts_including_trailer,
+                    "starts_including_trailer[{i}]"
+                );
+                assert_eq!(has_trailer.value(i), block.has_trailer, "has_trailer[{i}]");
+                let derived_preamble = i > 0 && has_trailer.value(i - 1);
+                assert_eq!(derived_preamble, block.has_preamble, "has_preamble[{i}]");
+            }
+        }
+    }
+}
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/layout.rs b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs
new file mode 100644
index 00000000000..f9749219ea6
--- /dev/null
+++ b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use lance_core::Result;
+
+use crate::{repdef::MiniBlockRepDefBudget, version::LanceFileVersion};
+
+/// Runs automatic sparse planning only after the dense mini-block budget makes it useful.
+pub(super) fn select_automatic_sparse<T>(
+    version: LanceFileVersion,
+    requested_encoding: Option<&str>,
+    dense_budget: &MiniBlockRepDefBudget,
+    candidate: impl FnOnce() -> Result<Option<T>>,
+) -> Result<Option<T>> {
+    if version < LanceFileVersion::V2_3
+        || requested_encoding.is_some()
+        || !matches!(
+            dense_budget,
+            MiniBlockRepDefBudget::RequiresPageSplit(_)
+                | MiniBlockRepDefBudget::SingleRowOverBudget(_)
+        )
+    {
+        return Ok(None);
+    }
+    candidate()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::constants::{
+        STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE,
+    };
+
+    #[test]
+    fn within_budget_does_not_construct_sparse_candidate() {
+        let selected = select_automatic_sparse::<()>(
+            LanceFileVersion::V2_3,
+            None,
+            &MiniBlockRepDefBudget::WithinBudget,
+            || panic!("within-budget pages must not construct sparse candidates"),
+        )
+        .unwrap();
+        assert!(selected.is_none());
+    }
+
+    #[test]
+    fn over_budget_selects_only_eligible_candidates() {
+        let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new());
+        let selected =
+            select_automatic_sparse(LanceFileVersion::V2_3, None, &split, || Ok(Some(42))).unwrap();
+        assert_eq!(selected, Some(42));
+
+        let ineligible =
+            select_automatic_sparse::<()>(LanceFileVersion::V2_3, None, &split, || Ok(None))
+                .unwrap();
+        assert!(ineligible.is_none());
+
+        let unsplittable = MiniBlockRepDefBudget::SingleRowOverBudget(70_000);
+        let selected =
+            select_automatic_sparse(LanceFileVersion::V2_3, None, &unsplittable, || Ok(Some(7)))
+                .unwrap();
+        assert_eq!(selected, Some(7));
+    }
+
+    #[test]
+    fn explicit_modes_and_lance_2_2_do_not_auto_select() {
+        let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new());
+        for requested in [
+            STRUCTURAL_ENCODING_MINIBLOCK,
+            STRUCTURAL_ENCODING_FULLZIP,
+            STRUCTURAL_ENCODING_SPARSE,
+        ] {
+            let selected = select_automatic_sparse::<()>(
+                LanceFileVersion::V2_3,
+                Some(requested),
+                &split,
+                || panic!("explicit modes must not invoke automatic sparse planning"),
+            )
+            .unwrap();
+            assert!(selected.is_none());
+        }
+
+        let selected = select_automatic_sparse::<()>(LanceFileVersion::V2_2, None, &split, || {
+            panic!("Lance 2.2 must not invoke automatic sparse planning")
+        })
+        .unwrap();
+        assert!(selected.is_none());
+    }
+}
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
index 1cf3b9bf581..edfba526670 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
@@ -19,7 +19,8 @@ use lance_core::Result;
 pub const MAX_MINIBLOCK_BYTES: u64 = 8 * 1024 - 6;
 
 const DEFAULT_MAX_MINIBLOCK_VALUES: u64 = 4096;
-const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768;
+/// Maximum number of values that any mini-block decoder accepts from page metadata.
+pub(crate) const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768;
 
 fn parse_max_miniblock_values() -> u64 {
     let val = std::env::var("LANCE_MINIBLOCK_MAX_VALUES")
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs
new file mode 100644
index 00000000000..68becc0014f
--- /dev/null
+++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs
@@ -0,0 +1,5614 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use super::*;
+use arrow_array::new_empty_array;
+use arrow_buffer::ArrowNativeType;
+
+fn invalid_enum<T>(
+    value: std::result::Result<T, prost::UnknownEnumValue>,
+    label: &str,
+) -> Result<T> {
+    value.map_err(|error| {
+        Error::invalid_input_source(
+            format!("Sparse structural {label} has an invalid enum value: {error}").into(),
+        )
+    })
+}
+
+pub(super) mod writer;
+
+fn usize_from_u64(value: u64, label: &str) -> Result<usize> {
+    usize::try_from(value).map_err(|_| {
+        Error::invalid_input_source(
+            format!("Sparse structural {label} {value} exceeds usize::MAX").into(),
+        )
+    })
+}
+
+/// Native sparse structural representation used by the 2.3 sparse layout.
+///
+/// Layers are stored from outer-most to inner-most, matching the order Arrow structural
+/// encoders record offsets and validity.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SparseStructuralPlan {
+    pub(crate) layers: Vec<SparseStructuralLayerPlan>,
+    pub(crate) num_items: u64,
+    pub(crate) num_visible_items: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SparsePositionSet {
+    Empty,
+    All { len: u64 },
+    Range { start: u64, len: u64 },
+    Explicit(Vec<u64>),
+}
+
+impl SparsePositionSet {
+    pub(crate) fn from_positions(
+        positions: Vec<u64>,
+        domain_len: u64,
+        label: &str,
+    ) -> Result<Self> {
+        if positions.is_empty() {
+            return Ok(Self::Empty);
+        }
+        for window in positions.windows(2) {
+            let [previous, current] = window else {
+                continue;
+            };
+            if previous >= current {
+                return Err(Error::invalid_input_source(
+                    format!("Sparse structural {label} positions must be strictly increasing")
+                        .into(),
+                ));
+            }
+        }
+        if let Some(position) = positions.iter().find(|position| **position >= domain_len) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} position {} is outside layer with {} slots",
+                    position, domain_len
+                )
+                .into(),
+            ));
+        }
+
+        let len = u64::try_from(positions.len()).map_err(|_| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} position count exceeds u64::MAX").into(),
+            )
+        })?;
+        let first = positions.first().copied().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} positions are unexpectedly empty").into(),
+            )
+        })?;
+        let last = positions.last().copied().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} positions are unexpectedly empty").into(),
+            )
+        })?;
+        if first == 0 && len == domain_len && domain_len > 0 && last == domain_len - 1 {
+            return Ok(Self::All { len: domain_len });
+        }
+        if last - first + 1 == len {
+            return Ok(Self::Range { start: first, len });
+        }
+        Ok(Self::Explicit(positions))
+    }
+
+    pub(crate) fn empty() -> Self {
+        Self::Empty
+    }
+
+    pub(crate) fn all(len: u64) -> Self {
+        if len == 0 {
+            Self::Empty
+        } else {
+            Self::All { len }
+        }
+    }
+
+    pub(crate) fn range(start: u64, len: u64) -> Self {
+        if len == 0 {
+            Self::Empty
+        } else {
+            Self::Range { start, len }
+        }
+    }
+
+    pub(crate) fn len(&self) -> u64 {
+        match self {
+            Self::Empty => 0,
+            Self::All { len } | Self::Range { len, .. } => *len,
+            Self::Explicit(positions) => positions.len() as u64,
+        }
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub(crate) fn deep_size(&self) -> usize {
+        match self {
+            Self::Explicit(positions) => positions.len() * std::mem::size_of::<u64>(),
+            Self::Empty | Self::All { .. } | Self::Range { .. } => 0,
+        }
+    }
+
+    pub(crate) fn materialize(&self) -> Result<Vec<u64>> {
+        match self {
+            Self::Empty => Ok(Vec::new()),
+            Self::All { len } => Self::materialize_range(0, *len),
+            Self::Range { start, len } => Self::materialize_range(*start, *len),
+            Self::Explicit(positions) => Ok(positions.clone()),
+        }
+    }
+
+    fn materialize_range(start: u64, len: u64) -> Result<Vec<u64>> {
+        let len_usize = usize::try_from(len).map_err(|_| {
+            Error::invalid_input_source(
+                format!(
+                    "Sparse structural position range length {} exceeds usize::MAX",
+                    len
+                )
+                .into(),
+            )
+        })?;
+        let end = start.checked_add(len).ok_or_else(|| {
+            Error::invalid_input_source("Sparse structural position range overflows".into())
+        })?;
+        let mut positions = Vec::with_capacity(len_usize);
+        positions.extend(start..end);
+        Ok(positions)
+    }
+
+    fn contains(&self, position: u64) -> bool {
+        match self {
+            Self::Empty => false,
+            Self::All { len } => position < *len,
+            Self::Range { start, len } => {
+                position >= *start && position < start.saturating_add(*len)
+            }
+            Self::Explicit(positions) => positions.binary_search(&position).is_ok(),
+        }
+    }
+
+    fn is_subset_of(&self, other: &Self, domain_len: u64) -> Result<bool> {
+        self.validate_domain(domain_len, "subset")?;
+        other.validate_domain(domain_len, "superset")?;
+        Ok(match self {
+            Self::Empty => true,
+            Self::All { .. } => other.len() == domain_len,
+            Self::Range { start, len } => {
+                let end = start.checked_add(*len).ok_or_else(|| {
+                    Error::invalid_input_source("Sparse structural subset range overflows".into())
+                })?;
+                match other {
+                    Self::All { .. } => true,
+                    Self::Range {
+                        start: other_start,
+                        len: other_len,
+                    } => {
+                        let other_end = other_start.saturating_add(*other_len);
+                        *start >= *other_start && end <= other_end
+                    }
+                    Self::Explicit(positions) => {
+                        let first = positions.partition_point(|position| *position < *start);
+                        let last = positions.partition_point(|position| *position < end);
+                        u64::try_from(last.saturating_sub(first)).ok() == Some(*len)
+                    }
+                    Self::Empty => false,
+                }
+            }
+            Self::Explicit(positions) => positions.iter().all(|position| other.contains(*position)),
+        })
+    }
+
+    fn is_disjoint(&self, other: &Self, domain_len: u64) -> Result<bool> {
+        self.validate_domain(domain_len, "first disjoint set")?;
+        other.validate_domain(domain_len, "second disjoint set")?;
+        let (smaller, larger) = if self.len() <= other.len() {
+            (self, other)
+        } else {
+            (other, self)
+        };
+        Ok(match smaller {
+            Self::Empty => true,
+            Self::All { .. } => larger.is_empty(),
+            Self::Range { start, len } => {
+                let end = start.saturating_add(*len);
+                match larger {
+                    Self::Empty => true,
+                    Self::All { .. } => false,
+                    Self::Range {
+                        start: other_start,
+                        len: other_len,
+                    } => end <= *other_start || other_start.saturating_add(*other_len) <= *start,
+                    Self::Explicit(positions) => {
+                        let index = positions.partition_point(|position| *position < *start);
+                        positions.get(index).is_none_or(|position| *position >= end)
+                    }
+                }
+            }
+            Self::Explicit(positions) => {
+                positions.iter().all(|position| !larger.contains(*position))
+            }
+        })
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SparseValidityMeaning {
+    NullPositions,
+    ValidPositions,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SparseValiditySet {
+    pub(crate) meaning: SparseValidityMeaning,
+    pub(crate) positions: SparsePositionSet,
+}
+
+impl SparseValiditySet {
+    pub(crate) fn deep_size(&self) -> usize {
+        self.positions.deep_size()
+    }
+
+    fn contains_only_valid_positions(
+        &self,
+        positions: &SparsePositionSet,
+        num_slots: u64,
+    ) -> Result<bool> {
+        match self.meaning {
+            SparseValidityMeaning::NullPositions => {
+                positions.is_disjoint(&self.positions, num_slots)
+            }
+            SparseValidityMeaning::ValidPositions => {
+                positions.is_subset_of(&self.positions, num_slots)
+            }
+        }
+    }
+
+    fn append_to(&self, validity: &mut BooleanBufferBuilder, num_slots: u64) -> Result<()> {
+        self.positions.validate_domain(num_slots, "validity")?;
+        let num_slots_usize = usize_from_u64(num_slots, "validity slot count")?;
+        match (self.meaning, &self.positions) {
+            (SparseValidityMeaning::NullPositions, SparsePositionSet::Empty) => {
+                validity.append_n(num_slots_usize, true);
+            }
+            (SparseValidityMeaning::ValidPositions, SparsePositionSet::Empty) => {
+                validity.append_n(num_slots_usize, false);
+            }
+            (SparseValidityMeaning::NullPositions, SparsePositionSet::All { .. }) => {
+                validity.append_n(num_slots_usize, false);
+            }
+            (SparseValidityMeaning::ValidPositions, SparsePositionSet::All { .. }) => {
+                validity.append_n(num_slots_usize, true);
+            }
+            (meaning, SparsePositionSet::Range { start, len }) => {
+                let range_end = start.checked_add(*len).ok_or_else(|| {
+                    Error::invalid_input_source("Sparse structural validity range overflows".into())
+                })?;
+                let default_valid = matches!(meaning, SparseValidityMeaning::NullPositions);
+                let range_valid = !default_valid;
+                validity.append_n(
+                    usize_from_u64(*start, "validity range start")?,
+                    default_valid,
+                );
+                validity.append_n(usize_from_u64(*len, "validity range length")?, range_valid);
+                validity.append_n(
+                    usize_from_u64(num_slots - range_end, "validity range tail")?,
+                    default_valid,
+                );
+            }
+            (_, SparsePositionSet::Explicit(_)) => {
+                let mut cursor = SparseValidityCursor::new(self, num_slots, "validity")?;
+                for slot in 0..num_slots {
+                    validity.append(cursor.is_valid(slot)?);
+                }
+                cursor.finish()?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl SparsePositionSet {
+    fn validate_domain(&self, domain_len: u64, label: &str) -> Result<()> {
+        match self {
+            Self::Empty => {}
+            Self::All { len } => {
+                if *len != domain_len {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} all set length {} does not match domain {}",
+                            len, domain_len
+                        )
+                        .into(),
+                    ));
+                }
+            }
+            Self::Range { start, len } => {
+                let end = start.checked_add(*len).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} range overflows").into(),
+                    )
+                })?;
+                if end > domain_len {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} range {}..{} is outside domain {}",
+                            start, end, domain_len
+                        )
+                        .into(),
+                    ));
+                }
+            }
+            Self::Explicit(positions) => {
+                for window in positions.windows(2) {
+                    let [previous, current] = window else {
+                        continue;
+                    };
+                    if previous >= current {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural {label} positions must be strictly increasing"
+                            )
+                            .into(),
+                        ));
+                    }
+                }
+                if let Some(position) = positions.last()
+                    && *position >= domain_len
+                {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} position {} is outside layer with {} slots",
+                            position, domain_len
+                        )
+                        .into(),
+                    ));
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+struct SparsePositionSetCursor<'a> {
+    set: &'a SparsePositionSet,
+    explicit: Option<std::iter::Peekable<std::slice::Iter<'a, u64>>>,
+}
+
+impl<'a> SparsePositionSetCursor<'a> {
+    fn new(set: &'a SparsePositionSet, domain_len: u64, label: &str) -> Result<Self> {
+        set.validate_domain(domain_len, label)?;
+        let explicit = match set {
+            SparsePositionSet::Explicit(positions) => Some(positions.iter().peekable()),
+            _ => None,
+        };
+        Ok(Self { set, explicit })
+    }
+
+    fn contains(&mut self, slot: u64) -> Result<bool> {
+        Ok(match self.set {
+            SparsePositionSet::Empty => false,
+            SparsePositionSet::All { .. } => true,
+            SparsePositionSet::Range { start, len } => {
+                slot >= *start && slot < start.saturating_add(*len)
+            }
+            SparsePositionSet::Explicit(_) => {
+                let iter = self.explicit.as_mut().ok_or_else(|| {
+                    Error::internal("Sparse structural explicit cursor is missing".to_string())
+                })?;
+                if let Some(position) = iter.peek()
+                    && **position < slot
+                {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural explicit position {} was skipped before slot {}",
+                            **position, slot
+                        )
+                        .into(),
+                    ));
+                }
+                if iter.peek().is_some_and(|position| **position == slot) {
+                    iter.next();
+                    true
+                } else {
+                    false
+                }
+            }
+        })
+    }
+
+    fn finish(&mut self) -> Result<()> {
+        if let Some(iter) = self.explicit.as_mut()
+            && let Some(position) = iter.next()
+        {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural explicit position {} was not consumed",
+                    position
+                )
+                .into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+struct SparseValidityCursor<'a> {
+    meaning: SparseValidityMeaning,
+    positions: SparsePositionSetCursor<'a>,
+}
+
+impl<'a> SparseValidityCursor<'a> {
+    fn new(validity: &'a SparseValiditySet, domain_len: u64, label: &str) -> Result<Self> {
+        Ok(Self {
+            meaning: validity.meaning,
+            positions: SparsePositionSetCursor::new(&validity.positions, domain_len, label)?,
+        })
+    }
+
+    fn is_valid(&mut self, slot: u64) -> Result<bool> {
+        let is_stored = self.positions.contains(slot)?;
+        Ok(match self.meaning {
+            SparseValidityMeaning::NullPositions => !is_stored,
+            SparseValidityMeaning::ValidPositions => is_stored,
+        })
+    }
+
+    fn finish(&mut self) -> Result<()> {
+        self.positions.finish()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SparseCountSet {
+    Empty,
+    Constant {
+        value: u64,
+        len: u64,
+    },
+    Explicit {
+        counts: Arc<[u64]>,
+        offsets: Arc<[u64]>,
+    },
+}
+
+impl SparseCountSet {
+    pub(crate) fn from_counts(counts: Vec<u64>) -> Result<Self> {
+        if counts.is_empty() {
+            return Ok(Self::Empty);
+        }
+        if let Some(first) = counts.first().copied()
+            && counts.iter().all(|count| *count == first)
+        {
+            return Ok(Self::Constant {
+                value: first,
+                len: counts.len() as u64,
+            });
+        }
+        let offsets = offsets_from_counts(&counts)?;
+        Ok(Self::Explicit {
+            counts: counts.into(),
+            offsets: offsets.into(),
+        })
+    }
+
+    pub(crate) fn constant(value: u64, len: u64) -> Self {
+        if len == 0 {
+            Self::Empty
+        } else {
+            Self::Constant { value, len }
+        }
+    }
+
+    pub(crate) fn len(&self) -> u64 {
+        match self {
+            Self::Empty => 0,
+            Self::Constant { len, .. } => *len,
+            Self::Explicit { counts, .. } => counts.len() as u64,
+        }
+    }
+
+    pub(crate) fn deep_size(&self) -> usize {
+        match self {
+            Self::Explicit { counts, offsets } => {
+                counts.len() * std::mem::size_of::<u64>()
+                    + offsets.len() * std::mem::size_of::<u64>()
+            }
+            Self::Empty | Self::Constant { .. } => 0,
+        }
+    }
+
+    pub(crate) fn materialize(&self) -> Result<Vec<u64>> {
+        match self {
+            Self::Empty => Ok(Vec::new()),
+            Self::Constant { value, len } => {
+                let len = usize::try_from(*len).map_err(|_| {
+                    Error::invalid_input_source(
+                        "Sparse structural count set length exceeds usize::MAX".into(),
+                    )
+                })?;
+                Ok(vec![*value; len])
+            }
+            Self::Explicit { counts, .. } => Ok(counts.to_vec()),
+        }
+    }
+
+    pub(crate) fn sum(&self) -> Result<u64> {
+        match self {
+            Self::Empty => Ok(0),
+            Self::Constant { value, len } => value.checked_mul(*len).ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "Sparse structural constant count sum overflows: value={}, len={}",
+                        value, len
+                    )
+                    .into(),
+                )
+            }),
+            Self::Explicit { offsets, .. } => offsets.last().copied().ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse structural explicit count offsets are empty".into(),
+                )
+            }),
+        }
+    }
+
+    fn validate_positive(&self) -> Result<()> {
+        let has_zero = match self {
+            Self::Empty => false,
+            Self::Constant { value, .. } => *value == 0,
+            Self::Explicit { counts, .. } => counts.contains(&0),
+        };
+        if has_zero {
+            return Err(Error::invalid_input_source(
+                "Sparse structural non-empty list count is zero".into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SparseStructuralLayerPlan {
+    Validity {
+        num_slots: u64,
+        validity: SparseValiditySet,
+    },
+    List {
+        num_slots: u64,
+        num_child_slots: u64,
+        non_empty_positions: SparsePositionSet,
+        counts: SparseCountSet,
+        validity: SparseValiditySet,
+    },
+    FixedSizeList {
+        num_slots: u64,
+        dimension: u64,
+        validity: SparseValiditySet,
+    },
+}
+
+impl SparseStructuralPlan {
+    fn expected_num_items(
+        layers: &[SparseStructuralLayerPlan],
+        num_visible_items: u64,
+    ) -> Result<u64> {
+        layers.iter().try_fold(num_visible_items, |items, layer| {
+            let additional = match layer {
+                SparseStructuralLayerPlan::List {
+                    num_slots,
+                    non_empty_positions,
+                    ..
+                } => num_slots
+                    .checked_sub(non_empty_positions.len())
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural list has more non-empty positions than slots".into(),
+                        )
+                    })?,
+                SparseStructuralLayerPlan::Validity { .. }
+                | SparseStructuralLayerPlan::FixedSizeList { .. } => 0,
+            };
+            items.checked_add(additional).ok_or_else(|| {
+                Error::invalid_input_source("Sparse structural item count overflows".into())
+            })
+        })
+    }
+
+    fn validate(&self, row_domain: u64) -> Result<()> {
+        usize_from_u64(self.num_visible_items, "visible item count")?;
+        let expected_num_items = Self::expected_num_items(&self.layers, self.num_visible_items)?;
+        if self.num_items != expected_num_items {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural item count {} does not match the {} items implied by its layers",
+                    self.num_items, expected_num_items
+                )
+                .into(),
+            ));
+        }
+        let mut expected_slots = row_domain;
+        for (layer_index, layer) in self.layers.iter().enumerate() {
+            let (num_slots, num_child_slots, validity) = match layer {
+                SparseStructuralLayerPlan::Validity {
+                    num_slots,
+                    validity,
+                } => (*num_slots, *num_slots, validity),
+                SparseStructuralLayerPlan::List {
+                    num_slots,
+                    num_child_slots,
+                    non_empty_positions,
+                    counts,
+                    validity,
+                } => {
+                    counts.validate_positive()?;
+                    if non_empty_positions.len() != counts.len() {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural list layer {} has {} non-empty positions but {} counts",
+                                layer_index,
+                                non_empty_positions.len(),
+                                counts.len()
+                            )
+                            .into(),
+                        ));
+                    }
+                    if counts.sum()? != *num_child_slots {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural list layer {} count sum does not match {} child slots",
+                                layer_index, num_child_slots
+                            )
+                            .into(),
+                        ));
+                    }
+                    if !validity.contains_only_valid_positions(non_empty_positions, *num_slots)? {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural list layer {} contains a non-empty null slot",
+                                layer_index
+                            )
+                            .into(),
+                        ));
+                    }
+                    (*num_slots, *num_child_slots, validity)
+                }
+                SparseStructuralLayerPlan::FixedSizeList {
+                    num_slots,
+                    dimension,
+                    validity,
+                } => {
+                    if *dimension == 0 {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural fixed-size-list layer {} has dimension zero",
+                                layer_index
+                            )
+                            .into(),
+                        ));
+                    }
+                    let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural fixed-size-list child domain overflows".into(),
+                        )
+                    })?;
+                    (*num_slots, num_child_slots, validity)
+                }
+            };
+            usize_from_u64(num_slots, "layer slot count")?;
+            usize_from_u64(num_child_slots, "layer child slot count")?;
+            if num_slots != expected_slots {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural layer {} has {} slots, expected {}",
+                        layer_index, num_slots, expected_slots
+                    )
+                    .into(),
+                ));
+            }
+            validity.positions.validate_domain(num_slots, "validity")?;
+            expected_slots = num_child_slots;
+        }
+        if expected_slots != self.num_visible_items {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural terminal domain has {} slots, expected {} visible items",
+                    expected_slots, self.num_visible_items
+                )
+                .into(),
+            ));
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct SparseStructuralUnraveler {
+    layers: Vec<SparseStructuralLayerPlan>,
+    next_layer: usize,
+    pending_fixed_size_list: bool,
+}
+
+impl SparseStructuralUnraveler {
+    pub(crate) fn new(plan: SparseStructuralPlan) -> Self {
+        let next_layer = plan.layers.len();
+        Self {
+            layers: plan.layers,
+            next_layer,
+            pending_fixed_size_list: false,
+        }
+    }
+
+    fn current_layer(&self) -> Option<&SparseStructuralLayerPlan> {
+        self.next_layer
+            .checked_sub(1)
+            .and_then(|idx| self.layers.get(idx))
+    }
+
+    fn consume_current_layer(&mut self) -> Result<()> {
+        self.next_layer = self.next_layer.checked_sub(1).ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse structural metadata has fewer layers than the Arrow schema".into(),
+            )
+        })?;
+        Ok(())
+    }
+
+    pub(crate) fn ensure_exhausted(&self) -> Result<()> {
+        if self.pending_fixed_size_list {
+            return Err(Error::invalid_input_source(
+                "Sparse structural metadata has an unconsumed fixed-size-list layer".into(),
+            ));
+        }
+        if self.next_layer != 0 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural metadata has {} unconsumed layer(s)",
+                    self.next_layer
+                )
+                .into(),
+            ));
+        }
+        Ok(())
+    }
+
+    pub(crate) fn is_all_valid(&self) -> bool {
+        match self.current_layer() {
+            Some(SparseStructuralLayerPlan::Validity {
+                num_slots,
+                validity,
+            })
+            | Some(SparseStructuralLayerPlan::FixedSizeList {
+                num_slots,
+                validity,
+                ..
+            })
+            | Some(SparseStructuralLayerPlan::List {
+                num_slots,
+                validity,
+                ..
+            }) => match validity.meaning {
+                SparseValidityMeaning::NullPositions => validity.positions.is_empty(),
+                SparseValidityMeaning::ValidPositions => validity.positions.len() == *num_slots,
+            },
+            None => true,
+        }
+    }
+
+    pub(crate) fn max_lists(&self) -> Result<usize> {
+        match self.current_layer() {
+            Some(SparseStructuralLayerPlan::List { num_slots, .. }) => {
+                usize_from_u64(*num_slots, "list slot count")
+            }
+            _ => Ok(0),
+        }
+    }
+
+    pub(crate) fn skip_validity(&mut self) -> Result<()> {
+        match self.current_layer() {
+            Some(SparseStructuralLayerPlan::Validity { .. }) => {
+                if self.pending_fixed_size_list {
+                    return Err(Error::invalid_input_source(
+                        "Sparse fixed-size-list schema does not match a validity layer".into(),
+                    ));
+                }
+                self.consume_current_layer()?;
+            }
+            Some(SparseStructuralLayerPlan::FixedSizeList { .. }) => {
+                if !self.pending_fixed_size_list {
+                    return Err(Error::invalid_input_source(
+                        "Sparse fixed-size-list layer does not match the Arrow schema".into(),
+                    ));
+                }
+                self.pending_fixed_size_list = false;
+                self.consume_current_layer()?;
+            }
+            None => {
+                return Err(Error::invalid_input_source(
+                    "Sparse structural metadata has fewer layers than the Arrow schema".into(),
+                ));
+            }
+            Some(SparseStructuralLayerPlan::List { .. }) => {
+                return Err(Error::invalid_input_source(
+                    "Sparse structural list layer does not match an Arrow validity layer".into(),
+                ));
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> {
+        match self.current_layer() {
+            Some(SparseStructuralLayerPlan::Validity {
+                num_slots,
+                validity: layer_validity,
+            }) => {
+                if self.pending_fixed_size_list {
+                    return Err(Error::invalid_input_source(
+                        "Sparse fixed-size-list schema does not match a validity layer".into(),
+                    ));
+                }
+                layer_validity.append_to(validity, *num_slots)?;
+                self.consume_current_layer()?;
+            }
+            Some(SparseStructuralLayerPlan::FixedSizeList {
+                num_slots,
+                validity: layer_validity,
+                ..
+            }) => {
+                if !self.pending_fixed_size_list {
+                    return Err(Error::invalid_input_source(
+                        "Sparse fixed-size-list layer does not match the Arrow schema".into(),
+                    ));
+                }
+                layer_validity.append_to(validity, *num_slots)?;
+                self.pending_fixed_size_list = false;
+                self.consume_current_layer()?;
+            }
+            None => {
+                return Err(Error::invalid_input_source(
+                    "Sparse structural metadata has fewer layers than the Arrow schema".into(),
+                ));
+            }
+            Some(SparseStructuralLayerPlan::List { .. }) => {
+                return Err(Error::invalid_input_source(
+                    "Sparse structural list layer does not match an Arrow validity layer".into(),
+                ));
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn decimate(&mut self, dimension: usize) -> Result<()> {
+        if self.pending_fixed_size_list {
+            return Err(Error::invalid_input_source(
+                "Sparse fixed-size-list layer was decimated more than once".into(),
+            ));
+        }
+        let Some(SparseStructuralLayerPlan::FixedSizeList {
+            dimension: actual_dimension,
+            ..
+        }) = self.current_layer()
+        else {
+            return Err(Error::invalid_input_source(
+                "Sparse structural layer does not match an Arrow fixed-size-list layer".into(),
+            ));
+        };
+        if usize_from_u64(*actual_dimension, "fixed-size-list dimension")? != dimension {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural fixed-size-list dimension {} does not match Arrow dimension {}",
+                    actual_dimension, dimension
+                )
+                .into(),
+            ));
+        }
+        self.pending_fixed_size_list = true;
+        Ok(())
+    }
+
+    fn to_offset<T: ArrowNativeType>(value: u64) -> Result<T> {
+        let value = usize::try_from(value).map_err(|_| {
+            Error::invalid_input_source(
+                format!("Sparse structural offset {} exceeds usize::MAX", value).into(),
+            )
+        })?;
+        T::from_usize(value).ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse structural offset does not fit the Arrow offset type".into(),
+            )
+        })
+    }
+
+    pub(crate) fn unravel_offsets<T: ArrowNativeType>(
+        &mut self,
+        offsets: &mut Vec<T>,
+        validity: Option<&mut BooleanBufferBuilder>,
+    ) -> Result<()> {
+        let Some(SparseStructuralLayerPlan::List {
+            num_slots,
+            num_child_slots,
+            non_empty_positions,
+            counts,
+            validity: layer_validity,
+            ..
+        }) = self.current_layer()
+        else {
+            return Err(Error::invalid_input_source(
+                "Sparse structural layer does not match an Arrow list layer".into(),
+            ));
+        };
+        if self.pending_fixed_size_list {
+            return Err(Error::invalid_input_source(
+                "Sparse fixed-size-list schema does not match an Arrow list layer".into(),
+            ));
+        }
+
+        if non_empty_positions.len() != counts.len() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural list has {} non-empty positions but {} counts",
+                    non_empty_positions.len(),
+                    counts.len()
+                )
+                .into(),
+            ));
+        }
+        let actual_child_slots = counts.sum()?;
+        if actual_child_slots != *num_child_slots {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural list count sum {} does not match child slots {}",
+                    actual_child_slots, num_child_slots
+                )
+                .into(),
+            ));
+        }
+
+        let mut current_offset = offsets
+            .last()
+            .map(|offset| offset.as_usize() as u64)
+            .unwrap_or(0);
+        if offsets.is_empty() {
+            offsets.push(Self::to_offset(current_offset)?);
+        }
+
+        if non_empty_positions.is_empty() {
+            if let Some(validity) = validity {
+                layer_validity.append_to(validity, *num_slots)?;
+            }
+            let offset = Self::to_offset(current_offset)?;
+            let new_len = offsets
+                .len()
+                .checked_add(usize_from_u64(*num_slots, "list slot count")?)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse structural list offset length overflows usize".into(),
+                    )
+                })?;
+            offsets.resize(new_len, offset);
+            self.consume_current_layer()?;
+            return Ok(());
+        }
+
+        let non_empty_positions = non_empty_positions.materialize()?;
+        let counts = counts.materialize()?;
+        let mut non_empty_iter = non_empty_positions
+            .iter()
+            .copied()
+            .zip(counts.iter().copied())
+            .peekable();
+        let mut validity_cursor =
+            SparseValidityCursor::new(layer_validity, *num_slots, "list validity")?;
+        let mut validity = validity;
+        for slot in 0..*num_slots {
+            let is_valid = validity_cursor.is_valid(slot)?;
+            if let Some(validity) = validity.as_mut() {
+                validity.append(is_valid);
+            }
+
+            if non_empty_iter
+                .peek()
+                .is_some_and(|(non_empty_pos, _)| *non_empty_pos == slot)
+            {
+                let (_, count) = non_empty_iter.next().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse structural list position is missing its child count".into(),
+                    )
+                })?;
+                if !is_valid {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list slot {} is both invalid and non-empty",
+                            slot
+                        )
+                        .into(),
+                    ));
+                }
+                current_offset = current_offset.checked_add(count).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse structural list offsets overflow u64".into(),
+                    )
+                })?;
+            }
+            offsets.push(Self::to_offset(current_offset)?);
+        }
+        validity_cursor.finish()?;
+        if let Some((extra_pos, _)) = non_empty_iter.next() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural non-empty position {} is outside layer with {} slots",
+                    extra_pos, num_slots
+                )
+                .into(),
+            ));
+        }
+
+        self.consume_current_layer()?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone)]
+enum SparsePositionSetDecoder {
+    Empty,
+    All {
+        len: u64,
+    },
+    Range {
+        start: u64,
+        len: u64,
+    },
+    Explicit {
+        decompressor: Arc<dyn BlockDecompressor>,
+        encoding: CompressiveEncoding,
+        count: u64,
+        domain_len: u64,
+    },
+}
+
+#[derive(Debug, Clone)]
+enum SparseCountSetDecoder {
+    Empty,
+    Constant {
+        value: u64,
+        len: u64,
+    },
+    Explicit {
+        decompressor: Arc<dyn BlockDecompressor>,
+        encoding: CompressiveEncoding,
+        count: u64,
+    },
+}
+
+#[derive(Debug, Clone)]
+enum SparseLayerDecompressors {
+    Validity {
+        num_slots: u64,
+        validity: SparseValiditySetDecoder,
+    },
+    List {
+        num_slots: u64,
+        num_child_slots: u64,
+        non_empty_positions: SparsePositionSetDecoder,
+        counts: SparseCountSetDecoder,
+        validity: SparseValiditySetDecoder,
+    },
+    FixedSizeList {
+        num_slots: u64,
+        num_child_slots: u64,
+        dimension: u64,
+        validity: SparseValiditySetDecoder,
+    },
+}
+
+#[derive(Debug, Clone)]
+struct SparseValiditySetDecoder {
+    meaning: SparseValidityMeaning,
+    positions: SparsePositionSetDecoder,
+}
+
+#[derive(Debug)]
+struct SparseStructuralCacheableState {
+    chunk_meta: Vec<ChunkMeta>,
+    chunk_value_offsets: Arc<[u64]>,
+    plan: SparseStructuralPlan,
+    row_domain: u64,
+}
+
+impl DeepSizeOf for SparseStructuralCacheableState {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        let structural_size = self
+            .plan
+            .layers
+            .iter()
+            .map(|layer| match layer {
+                SparseStructuralLayerPlan::Validity { validity, .. } => validity.deep_size(),
+                SparseStructuralLayerPlan::List {
+                    non_empty_positions,
+                    counts,
+                    validity,
+                    ..
+                } => non_empty_positions.deep_size() + counts.deep_size() + validity.deep_size(),
+                SparseStructuralLayerPlan::FixedSizeList { validity, .. } => validity.deep_size(),
+            })
+            .sum::<usize>();
+        self.chunk_meta.len() * std::mem::size_of::<ChunkMeta>()
+            + self.chunk_value_offsets.len() * std::mem::size_of::<u64>()
+            + structural_size
+    }
+}
+
+impl CachedPageData for SparseStructuralCacheableState {
+    fn as_arc_any(self: Arc<Self>) -> Arc<dyn Any + Send + Sync + 'static> {
+        self
+    }
+}
+
+#[derive(Debug)]
+pub(super) struct SparseStructuralScheduler {
+    buffer_offsets_and_sizes: Vec<(u64, u64)>,
+    priority: u64,
+    row_domain: u64,
+    row_scale: u64,
+    num_items: u64,
+    num_visible_items: u64,
+    num_buffers: u64,
+    value_encoding: CompressiveEncoding,
+    value_decompressor: Arc<dyn MiniBlockDecompressor>,
+    layer_decompressors: Vec<SparseLayerDecompressors>,
+    data_type: DataType,
+    page_meta: Option<Arc<SparseStructuralCacheableState>>,
+    has_large_chunk: bool,
+}
+
+impl SparseStructuralScheduler {
+    fn require_layer(
+        layer: &pb21::SparseStructuralLayer,
+    ) -> Result<&pb21::sparse_structural_layer::Layer> {
+        layer.layer.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse structural layer is missing its layer variant".into(),
+            )
+        })
+    }
+
+    fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> Result<u64> {
+        Ok(match Self::require_layer(layer)? {
+            pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots,
+            pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots,
+            pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots,
+        })
+    }
+
+    fn layer_num_child_slots(layer: &pb21::SparseStructuralLayer) -> Result<u64> {
+        Ok(match Self::require_layer(layer)? {
+            pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots,
+            pb21::sparse_structural_layer::Layer::List(layer) => layer.num_child_slots,
+            pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer
+                .num_slots
+                .checked_mul(layer.dimension)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}",
+                            layer.num_slots, layer.dimension
+                        )
+                        .into(),
+                    )
+                })?,
+        })
+    }
+
+    pub(super) fn try_new(
+        buffer_offsets_and_sizes: &[(u64, u64)],
+        priority: u64,
+        encoded_row_domain: u64,
+        data_type: DataType,
+        layout: &pb21::SparseLayout,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<Self> {
+        let value_compression = layout.value_compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source("Sparse layout is missing value compression".into())
+        })?;
+        let value_buffer_count = Self::validate_value_encoding(value_compression)?;
+        if layout.num_buffers != value_buffer_count {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout declares {} value buffers, but its compression descriptor requires {}",
+                    layout.num_buffers, value_buffer_count
+                )
+                .into(),
+            ));
+        }
+        let row_domain = match layout.structural_layers.first() {
+            Some(layer) => Self::layer_num_slots(layer)?,
+            None => encoded_row_domain,
+        };
+        Self::validate_domain_chain(
+            &layout.structural_layers,
+            row_domain,
+            layout.num_items,
+            layout.num_visible_items,
+        )?;
+        let expected_buffers = 2 + Self::structural_buffer_count(&layout.structural_layers)?;
+        if buffer_offsets_and_sizes.len() != expected_buffers {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout has {} buffers, expected {}",
+                    buffer_offsets_and_sizes.len(),
+                    expected_buffers
+                )
+                .into(),
+            ));
+        }
+        Self::validate_page_buffers(buffer_offsets_and_sizes, layout.num_visible_items)?;
+        let row_scale =
+            layout
+                .structural_layers
+                .iter()
+                .try_fold(1_u64, |scale, layer| -> Result<u64> {
+                    match Self::require_layer(layer)? {
+                        pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => {
+                            scale.checked_mul(layer.dimension).ok_or_else(|| {
+                                Error::invalid_input_source(
+                                    "Sparse structural fixed-size-list row scale overflows".into(),
+                                )
+                            })
+                        }
+                        pb21::sparse_structural_layer::Layer::Validity(_)
+                        | pb21::sparse_structural_layer::Layer::List(_) => Ok(scale),
+                    }
+                })?;
+        let expected_encoded_row_domain = row_domain.checked_mul(row_scale).ok_or_else(|| {
+            Error::invalid_input_source("Sparse structural encoded row domain overflows".into())
+        })?;
+        if encoded_row_domain != expected_encoded_row_domain {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural encoded row domain {} does not match outer domain {} * fixed-size-list scale {}",
+                    encoded_row_domain, row_domain, row_scale
+                )
+                .into(),
+            ));
+        }
+        let value_decompressor = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            decompressors.create_miniblock_decompressor(value_compression, decompressors)
+        }))
+        .map_err(|_| {
+            Error::invalid_input_source(
+                "Sparse value compression descriptor caused decompressor construction to panic"
+                    .into(),
+            )
+        })?
+        .map_err(|error| {
+            Error::invalid_input_source(
+                format!("Sparse value decompressor construction failed: {error}").into(),
+            )
+        })?;
+        let layer_decompressors = layout
+            .structural_layers
+            .iter()
+            .map(|layer| Self::layer_decompressors(layer, decompressors))
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Self {
+            buffer_offsets_and_sizes: buffer_offsets_and_sizes.to_vec(),
+            priority,
+            row_domain,
+            row_scale,
+            num_items: layout.num_items,
+            num_visible_items: layout.num_visible_items,
+            num_buffers: layout.num_buffers,
+            value_encoding: value_compression.clone(),
+            value_decompressor: value_decompressor.into(),
+            layer_decompressors,
+            data_type,
+            page_meta: None,
+            has_large_chunk: layout.has_large_chunk,
+        })
+    }
+
+    fn validate_compression<'a>(
+        compression: &'a CompressiveEncoding,
+        label: &str,
+    ) -> Result<&'a CompressiveEncoding> {
+        compression.compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} is missing compression details").into(),
+            )
+        })?;
+        Ok(compression)
+    }
+
+    fn validate_buffer_compression(
+        compression: Option<&pb21::BufferCompression>,
+        label: &str,
+    ) -> Result<()> {
+        let Some(compression) = compression else {
+            return Ok(());
+        };
+        match invalid_enum(
+            pb21::CompressionScheme::try_from(compression.scheme),
+            "compression scheme",
+        )? {
+            pb21::CompressionScheme::CompressionAlgorithmUnspecified => {
+                Err(Error::invalid_input_source(
+                    format!("Sparse structural {label} buffer compression is unspecified").into(),
+                ))
+            }
+            pb21::CompressionScheme::CompressionAlgorithmLz4
+            | pb21::CompressionScheme::CompressionAlgorithmZstd => Ok(()),
+        }
+    }
+
+    fn encoding_contains_general(root: &CompressiveEncoding) -> bool {
+        use pb21::compressive_encoding::Compression;
+
+        let mut stack = vec![root];
+        while let Some(encoding) = stack.pop() {
+            let Some(compression) = encoding.compression.as_ref() else {
+                continue;
+            };
+            match compression {
+                Compression::General(_) => return true,
+                Compression::Variable(variable) => {
+                    stack.extend(variable.offsets.as_deref());
+                }
+                Compression::OutOfLineBitpacking(bitpacking) => {
+                    stack.extend(bitpacking.values.as_deref());
+                }
+                Compression::Fsst(fsst) => stack.extend(fsst.values.as_deref()),
+                Compression::Dictionary(dictionary) => {
+                    stack.extend(dictionary.indices.as_deref());
+                    stack.extend(dictionary.items.as_deref());
+                }
+                Compression::Rle(rle) => {
+                    stack.extend(rle.values.as_deref());
+                    stack.extend(rle.run_lengths.as_deref());
+                }
+                Compression::ByteStreamSplit(split) => {
+                    stack.extend(split.values.as_deref());
+                }
+                Compression::FixedSizeList(fsl) => stack.extend(fsl.values.as_deref()),
+                Compression::PackedStruct(packed) => stack.extend(packed.values.as_deref()),
+                Compression::VariablePackedStruct(packed) => {
+                    stack.extend(
+                        packed
+                            .fields
+                            .iter()
+                            .filter_map(|field| field.value.as_ref()),
+                    );
+                }
+                Compression::Flat(_)
+                | Compression::Constant(_)
+                | Compression::InlineBitpacking(_) => {}
+            }
+        }
+        false
+    }
+
+    fn require_encoding<'a>(
+        encoding: &'a Option<Box<CompressiveEncoding>>,
+        label: &str,
+    ) -> Result<&'a CompressiveEncoding> {
+        encoding.as_deref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} encoding is required").into(),
+            )
+        })
+    }
+
+    fn validate_flat(flat: &pb21::Flat, label: &str) -> Result<()> {
+        if flat.bits_per_value == 0 {
+            return Err(Error::invalid_input_source(
+                format!("Sparse structural {label} flat bit width is zero").into(),
+            ));
+        }
+        if flat.data.is_some() {
+            return Err(Error::invalid_input_source(
+                format!("Sparse structural {label} uses unsupported leaf buffer compression")
+                    .into(),
+            ));
+        }
+        Ok(())
+    }
+
+    fn validate_value_encoding(encoding: &CompressiveEncoding) -> Result<u64> {
+        use pb21::compressive_encoding::Compression;
+
+        let compression = encoding.compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse value compression is missing compression details".into(),
+            )
+        })?;
+        match compression {
+            Compression::Flat(flat) => {
+                Self::validate_flat(flat, "value")?;
+                Ok(1)
+            }
+            Compression::InlineBitpacking(bitpacking) => {
+                if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse inline bitpacking width {} is not supported",
+                            bitpacking.uncompressed_bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                if bitpacking.values.is_some() {
+                    return Err(Error::invalid_input_source(
+                        "Sparse inline bitpacking uses unsupported leaf buffer compression".into(),
+                    ));
+                }
+                Ok(1)
+            }
+            Compression::Variable(variable) => {
+                let offsets = variable.offsets.as_deref().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse variable compression is missing offsets".into(),
+                    )
+                })?;
+                let Some(Compression::Flat(offsets)) = offsets.compression.as_ref() else {
+                    return Err(Error::invalid_input_source(
+                        "Sparse variable offsets must use flat compression".into(),
+                    ));
+                };
+                Self::validate_flat(offsets, "variable offsets")?;
+                if !matches!(offsets.bits_per_value, 32 | 64) {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse variable offset width {} is not supported",
+                            offsets.bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                if variable.values.is_some() {
+                    return Err(Error::invalid_input_source(
+                        "Sparse variable values use unsupported leaf buffer compression".into(),
+                    ));
+                }
+                Ok(1)
+            }
+            Compression::Fsst(fsst) => {
+                if fsst.symbol_table.is_empty() {
+                    return Err(Error::invalid_input_source(
+                        "Sparse FSST compression has an empty symbol table".into(),
+                    ));
+                }
+                let values = Self::require_encoding(&fsst.values, "FSST values")?;
+                if !matches!(values.compression.as_ref(), Some(Compression::Variable(_))) {
+                    return Err(Error::invalid_input_source(
+                        "Sparse FSST values must use variable compression".into(),
+                    ));
+                }
+                Self::validate_value_encoding(values)
+            }
+            Compression::ByteStreamSplit(split) => {
+                let values = Self::require_encoding(&split.values, "byte-stream-split values")?;
+                let Some(Compression::Flat(flat)) = values.compression.as_ref() else {
+                    return Err(Error::invalid_input_source(
+                        "Sparse byte-stream-split values must use flat compression".into(),
+                    ));
+                };
+                Self::validate_flat(flat, "byte-stream-split values")?;
+                if !matches!(flat.bits_per_value, 32 | 64) {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse byte-stream-split width {} is not supported",
+                            flat.bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(1)
+            }
+            Compression::FixedSizeList(fsl) => Self::validate_fsl_value_encoding(fsl),
+            Compression::PackedStruct(packed) => Self::validate_packed_value_encoding(packed),
+            Compression::Rle(rle) => {
+                let values = Self::require_encoding(&rle.values, "RLE values")?;
+                let lengths = Self::require_encoding(&rle.run_lengths, "RLE run lengths")?;
+                Self::validate_block_encoding(values, "RLE values")?;
+                Self::validate_block_encoding(lengths, "RLE run lengths")?;
+                Ok(2)
+            }
+            Compression::General(general) => {
+                let compression = general.compression.as_ref().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse general compression is missing its buffer compression".into(),
+                    )
+                })?;
+                Self::validate_buffer_compression(Some(compression), "general")?;
+                Self::validate_value_encoding(Self::require_encoding(
+                    &general.values,
+                    "general values",
+                )?)
+            }
+            Compression::Constant(_)
+            | Compression::OutOfLineBitpacking(_)
+            | Compression::Dictionary(_)
+            | Compression::VariablePackedStruct(_) => Err(Error::invalid_input_source(
+                "Sparse value compression uses an unsupported mini-block encoding".into(),
+            )),
+        }
+    }
+
+    fn validate_fsl_value_encoding(fsl: &pb21::FixedSizeList) -> Result<u64> {
+        use pb21::compressive_encoding::Compression;
+
+        if fsl.items_per_value == 0 {
+            return Err(Error::invalid_input_source(
+                "Sparse fixed-size-list value compression has dimension zero".into(),
+            ));
+        }
+        let values = Self::require_encoding(&fsl.values, "fixed-size-list values")?;
+        let child_buffers = match values.compression.as_ref() {
+            Some(Compression::Flat(flat)) => {
+                Self::validate_flat(flat, "fixed-size-list values")?;
+                1_u64
+            }
+            Some(Compression::FixedSizeList(inner)) => Self::validate_fsl_value_encoding(inner)?,
+            _ => {
+                return Err(Error::invalid_input_source(
+                    "Sparse fixed-size-list values must use fixed-size-list or flat compression"
+                        .into(),
+                ));
+            }
+        };
+        child_buffers
+            .checked_add(u64::from(fsl.has_validity))
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse fixed-size-list value buffer count overflows".into(),
+                )
+            })
+    }
+
+    fn validate_packed_value_encoding(packed: &pb21::PackedStruct) -> Result<u64> {
+        use pb21::compressive_encoding::Compression;
+
+        if packed.bits_per_value.is_empty()
+            || packed
+                .bits_per_value
+                .iter()
+                .any(|bits| *bits == 0 || !bits.is_multiple_of(8))
+        {
+            return Err(Error::invalid_input_source(
+                "Sparse packed-struct widths must be non-empty positive byte widths".into(),
+            ));
+        }
+        let values = Self::require_encoding(&packed.values, "packed-struct values")?;
+        let Some(Compression::Flat(flat)) = values.compression.as_ref() else {
+            return Err(Error::invalid_input_source(
+                "Sparse packed-struct values must use flat compression".into(),
+            ));
+        };
+        Self::validate_flat(flat, "packed-struct values")?;
+        let total_bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| {
+            sum.checked_add(*bits).ok_or_else(|| {
+                Error::invalid_input_source("Sparse packed-struct bit width sum overflows".into())
+            })
+        })?;
+        if total_bits != flat.bits_per_value {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse packed-struct child widths sum to {}, but values use {} bits",
+                    total_bits, flat.bits_per_value
+                )
+                .into(),
+            ));
+        }
+        Ok(1)
+    }
+
+    fn validate_block_encoding(encoding: &CompressiveEncoding, label: &str) -> Result<()> {
+        use pb21::compressive_encoding::Compression;
+
+        match encoding.compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} is missing compression details").into(),
+            )
+        })? {
+            Compression::Flat(flat) => Self::validate_flat(flat, label),
+            Compression::InlineBitpacking(bitpacking) => {
+                if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} inline bitpacking width {} is unsupported",
+                            bitpacking.uncompressed_bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                if bitpacking.values.is_some() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} uses unsupported leaf buffer compression"
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(())
+            }
+            Compression::OutOfLineBitpacking(bitpacking) => {
+                if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} out-of-line bitpacking width {} is unsupported",
+                            bitpacking.uncompressed_bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                let values = Self::require_encoding(&bitpacking.values, label)?;
+                let Some(Compression::Flat(flat)) = values.compression.as_ref() else {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} bitpacked values must be flat").into(),
+                    ));
+                };
+                Self::validate_flat(flat, label)
+            }
+            Compression::Constant(constant) => {
+                if constant
+                    .value
+                    .as_ref()
+                    .is_some_and(|value| value.len() != 8)
+                {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} constant must be 64 bits").into(),
+                    ));
+                }
+                Ok(())
+            }
+            Compression::General(general) => {
+                let compression = general.compression.as_ref().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} general compression is missing config")
+                            .into(),
+                    )
+                })?;
+                Self::validate_buffer_compression(Some(compression), label)?;
+                Self::validate_block_encoding(
+                    Self::require_encoding(&general.values, label)?,
+                    label,
+                )
+            }
+            Compression::Rle(rle) => {
+                Self::validate_block_encoding(
+                    Self::require_encoding(&rle.values, "RLE values")?,
+                    "RLE values",
+                )?;
+                Self::validate_block_encoding(
+                    Self::require_encoding(&rle.run_lengths, "RLE run lengths")?,
+                    "RLE run lengths",
+                )
+            }
+            _ => Err(Error::invalid_input_source(
+                format!("Sparse structural {label} uses an unsupported block encoding").into(),
+            )),
+        }
+    }
+
+    fn validate_domain_chain(
+        layers: &[pb21::SparseStructuralLayer],
+        row_domain: u64,
+        num_items: u64,
+        num_visible_items: u64,
+    ) -> Result<()> {
+        let expected_num_items =
+            layers
+                .iter()
+                .try_fold(num_visible_items, |items, layer| -> Result<u64> {
+                    let additional = match Self::require_layer(layer)? {
+                        pb21::sparse_structural_layer::Layer::List(layer) => {
+                            let positions = Self::require_position_set(
+                                &layer.non_empty_positions,
+                                "list non-empty",
+                            )?;
+                            let num_non_empty = Self::position_cardinality(
+                                positions,
+                                layer.num_slots,
+                                "list non-empty positions",
+                            )?;
+                            layer.num_slots.checked_sub(num_non_empty).ok_or_else(|| {
+                                Error::invalid_input_source(
+                                "Sparse structural list has more non-empty positions than slots"
+                                    .into(),
+                            )
+                            })?
+                        }
+                        pb21::sparse_structural_layer::Layer::Validity(_)
+                        | pb21::sparse_structural_layer::Layer::FixedSizeList(_) => 0,
+                    };
+                    items.checked_add(additional).ok_or_else(|| {
+                        Error::invalid_input_source("Sparse structural item count overflows".into())
+                    })
+                })?;
+        if num_items != expected_num_items {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout has {} structural items, but its layers imply {}",
+                    num_items, expected_num_items
+                )
+                .into(),
+            ));
+        }
+        let mut expected_slots = row_domain;
+        for (layer_index, layer) in layers.iter().enumerate() {
+            let num_slots = Self::layer_num_slots(layer)?;
+            let num_child_slots = Self::layer_num_child_slots(layer)?;
+            usize_from_u64(num_slots, "layer slot count")?;
+            usize_from_u64(num_child_slots, "layer child slot count")?;
+            if num_slots != expected_slots {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural layer {} has {} slots, expected {} from the outer domain",
+                        layer_index, num_slots, expected_slots
+                    )
+                    .into(),
+                ));
+            }
+            expected_slots = num_child_slots;
+        }
+        if expected_slots != num_visible_items {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural terminal domain has {} slots, expected {} visible items",
+                    expected_slots, num_visible_items
+                )
+                .into(),
+            ));
+        }
+        Ok(())
+    }
+
+    fn metadata_buffer(&self) -> Result<(u64, u64)> {
+        self.buffer_offsets_and_sizes
+            .first()
+            .copied()
+            .ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout is missing metadata buffer".into())
+            })
+    }
+
+    fn value_buffer(&self) -> Result<(u64, u64)> {
+        self.buffer_offsets_and_sizes
+            .get(1)
+            .copied()
+            .ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout is missing value buffer".into())
+            })
+    }
+
+    fn checked_buffer_range(position: u64, size: u64, label: &str) -> Result<Range<u64>> {
+        let end = position.checked_add(size).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} buffer range overflows").into(),
+            )
+        })?;
+        Ok(position..end)
+    }
+
+    fn validate_page_buffers(
+        buffer_offsets_and_sizes: &[(u64, u64)],
+        num_visible_items: u64,
+    ) -> Result<()> {
+        let (_, metadata_size) = buffer_offsets_and_sizes.first().copied().ok_or_else(|| {
+            Error::invalid_input_source("Sparse layout is missing metadata buffer".into())
+        })?;
+        if !metadata_size.is_multiple_of(8) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout metadata buffer has {metadata_size} bytes, which is not a multiple of 8"
+                )
+                .into(),
+            ));
+        }
+        let chunk_count = metadata_size / 8;
+        if num_visible_items == 0 {
+            if chunk_count != 0 {
+                return Err(Error::invalid_input_source(
+                    "Sparse layout with no visible items must have empty chunk metadata".into(),
+                ));
+            }
+        } else {
+            let min_chunks =
+                num_visible_items.div_ceil(miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES);
+            if chunk_count < min_chunks || chunk_count > num_visible_items {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse layout metadata declares {chunk_count} chunks for {num_visible_items} visible items, expected {min_chunks}..={num_visible_items}"
+                    )
+                    .into(),
+                ));
+            }
+        }
+
+        let (_, value_size) = buffer_offsets_and_sizes.get(1).copied().ok_or_else(|| {
+            Error::invalid_input_source("Sparse layout is missing value buffer".into())
+        })?;
+        if (num_visible_items == 0) != (value_size == 0) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout value buffer has {value_size} bytes for {num_visible_items} visible items"
+                )
+                .into(),
+            ));
+        }
+
+        for (position, size) in buffer_offsets_and_sizes.iter().skip(2) {
+            Self::checked_buffer_range(*position, *size, "structural")?;
+        }
+
+        for (position, size) in buffer_offsets_and_sizes.iter().take(2) {
+            Self::checked_buffer_range(*position, *size, "page")?;
+        }
+        Ok(())
+    }
+
+    fn require_position_set<'a>(
+        set: &'a Option<pb21::SparsePositionSet>,
+        label: &str,
+    ) -> Result<&'a pb21::SparsePositionSet> {
+        set.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} position set is required").into(),
+            )
+        })
+    }
+
+    fn require_count_set<'a>(
+        set: &'a Option<pb21::SparseCountSet>,
+        label: &str,
+    ) -> Result<&'a pb21::SparseCountSet> {
+        set.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} count set is required").into(),
+            )
+        })
+    }
+
+    fn require_validity_set<'a>(
+        set: &'a Option<pb21::SparseValiditySet>,
+        label: &str,
+    ) -> Result<&'a pb21::SparseValiditySet> {
+        set.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} validity set is required").into(),
+            )
+        })
+    }
+
+    fn validity_meaning(
+        validity_set: &pb21::SparseValiditySet,
+        label: &str,
+    ) -> Result<SparseValidityMeaning> {
+        match invalid_enum(
+            pb21::sparse_validity_set::Meaning::try_from(validity_set.meaning),
+            "validity meaning",
+        )? {
+            pb21::sparse_validity_set::Meaning::SparseValidityUnspecified => {
+                Err(Error::invalid_input_source(
+                    format!("Sparse structural {label} meaning is unspecified").into(),
+                ))
+            }
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions => {
+                Ok(SparseValidityMeaning::NullPositions)
+            }
+            pb21::sparse_validity_set::Meaning::SparseValidityValidPositions => {
+                Ok(SparseValidityMeaning::ValidPositions)
+            }
+        }
+    }
+
+    fn validity_buffer_count(
+        validity_set: &pb21::SparseValiditySet,
+        domain_len: u64,
+        label: &str,
+    ) -> Result<(usize, SparseValidityMeaning, u64)> {
+        let meaning = Self::validity_meaning(validity_set, label)?;
+        let position_set = validity_set.positions.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} positions are required").into(),
+            )
+        })?;
+        let cardinality = Self::position_cardinality(position_set, domain_len, label)?;
+        let buffer_count = Self::position_buffer_count(position_set, domain_len, label)?;
+        Ok((buffer_count, meaning, cardinality))
+    }
+
+    fn position_cardinality(
+        position_set: &pb21::SparsePositionSet,
+        domain_len: u64,
+        label: &str,
+    ) -> Result<u64> {
+        let positions = position_set.positions.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} position set is missing its variant").into(),
+            )
+        })?;
+        let cardinality = position_set.num_positions;
+        if cardinality > domain_len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} cardinality {} exceeds domain {}",
+                    cardinality, domain_len
+                )
+                .into(),
+            ));
+        }
+        match positions {
+            pb21::sparse_position_set::Positions::Empty(_) => {
+                if cardinality != 0 {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} empty set has cardinality {}",
+                            cardinality
+                        )
+                        .into(),
+                    ));
+                }
+            }
+            pb21::sparse_position_set::Positions::All(_) => {
+                if domain_len == 0 || cardinality != domain_len {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} all set has cardinality {}, expected {}",
+                            cardinality, domain_len
+                        )
+                        .into(),
+                    ));
+                }
+            }
+            pb21::sparse_position_set::Positions::Range(range) => {
+                let end = range.start.checked_add(range.length).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} range overflows").into(),
+                    )
+                })?;
+                if range.length == 0 || range.length != cardinality || end > domain_len {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} range {}..{} does not match cardinality {} in domain {}",
+                            range.start, end, cardinality, domain_len
+                        )
+                        .into(),
+                    ));
+                }
+            }
+            pb21::sparse_position_set::Positions::Explicit(compression) => {
+                if cardinality == 0 {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} has compression but no values").into(),
+                    ));
+                }
+                Self::validate_compression(compression, label)?;
+            }
+        }
+        Ok(cardinality)
+    }
+
+    fn position_buffer_count(
+        position_set: &pb21::SparsePositionSet,
+        domain_len: u64,
+        label: &str,
+    ) -> Result<usize> {
+        Self::position_cardinality(position_set, domain_len, label)?;
+        let positions = position_set.positions.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} position set is missing its variant").into(),
+            )
+        })?;
+        Ok(usize::from(matches!(
+            positions,
+            pb21::sparse_position_set::Positions::Explicit(_)
+        )))
+    }
+
+    fn count_buffer_count(
+        count_set: &pb21::SparseCountSet,
+        cardinality: u64,
+        label: &str,
+    ) -> Result<usize> {
+        let counts = count_set.counts.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} count set is missing its variant").into(),
+            )
+        })?;
+        match counts {
+            pb21::sparse_count_set::Counts::Empty(_) => {
+                if cardinality != 0 {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} empty count set has cardinality {}",
+                            cardinality
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(0)
+            }
+            pb21::sparse_count_set::Counts::Constant(constant) => {
+                if cardinality == 0 {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} constant count has no values").into(),
+                    ));
+                }
+                if constant.value == 0 {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} constant count is zero").into(),
+                    ));
+                }
+                Ok(0)
+            }
+            pb21::sparse_count_set::Counts::Explicit(compression) => {
+                if cardinality == 0 {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} has compression but no values").into(),
+                    ));
+                }
+                Self::validate_compression(compression, label)?;
+                Ok(1)
+            }
+        }
+    }
+
+    fn count_set_child_slots(
+        count_set: &pb21::SparseCountSet,
+        cardinality: u64,
+        label: &str,
+    ) -> Result<Option<u64>> {
+        let counts = count_set.counts.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} count set is missing its variant").into(),
+            )
+        })?;
+        match counts {
+            pb21::sparse_count_set::Counts::Empty(_) => Ok(Some(0)),
+            pb21::sparse_count_set::Counts::Constant(constant) => constant
+                .value
+                .checked_mul(cardinality)
+                .map(Some)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} constant count sum overflows: value={}, len={}",
+                            constant.value, cardinality
+                        )
+                        .into(),
+                    )
+                }),
+            pb21::sparse_count_set::Counts::Explicit(_) => Ok(None),
+        }
+    }
+
+    fn create_position_decompressor(
+        compression: &CompressiveEncoding,
+        label: &str,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<Arc<dyn BlockDecompressor>> {
+        let compression = Self::validate_compression(compression, label)?;
+        Self::validate_block_encoding(compression, label)?;
+        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            decompressors.create_block_decompressor(compression)
+        }))
+        .map_err(|_| {
+            Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} descriptor caused decompressor construction to panic"
+                )
+                .into(),
+            )
+        })?
+        .map(Arc::from)
+        .map_err(|error| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} decompressor construction failed: {error}")
+                    .into(),
+            )
+        })
+    }
+
+    fn position_set_decoder(
+        position_set: &pb21::SparsePositionSet,
+        domain_len: u64,
+        label: &str,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<(SparsePositionSetDecoder, u64)> {
+        let cardinality = Self::position_cardinality(position_set, domain_len, label)?;
+        let positions = position_set.positions.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} position set is missing its variant").into(),
+            )
+        })?;
+        Ok((
+            match positions {
+                pb21::sparse_position_set::Positions::Empty(_) => SparsePositionSetDecoder::Empty,
+                pb21::sparse_position_set::Positions::All(_) => {
+                    SparsePositionSetDecoder::All { len: domain_len }
+                }
+                pb21::sparse_position_set::Positions::Range(range) => {
+                    SparsePositionSetDecoder::Range {
+                        start: range.start,
+                        len: range.length,
+                    }
+                }
+                pb21::sparse_position_set::Positions::Explicit(compression) => {
+                    SparsePositionSetDecoder::Explicit {
+                        decompressor: Self::create_position_decompressor(
+                            compression,
+                            label,
+                            decompressors,
+                        )?,
+                        encoding: compression.clone(),
+                        count: cardinality,
+                        domain_len,
+                    }
+                }
+            },
+            cardinality,
+        ))
+    }
+
+    fn validity_set_decoder(
+        validity_set: &pb21::SparseValiditySet,
+        domain_len: u64,
+        label: &str,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<(SparseValiditySetDecoder, u64)> {
+        let meaning = Self::validity_meaning(validity_set, label)?;
+        let position_set = validity_set.positions.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} positions are required").into(),
+            )
+        })?;
+        let (positions, cardinality) =
+            Self::position_set_decoder(position_set, domain_len, label, decompressors)?;
+        Ok((SparseValiditySetDecoder { meaning, positions }, cardinality))
+    }
+
+    fn num_valid_slots(
+        meaning: SparseValidityMeaning,
+        cardinality: u64,
+        num_slots: u64,
+        label: &str,
+    ) -> Result<u64> {
+        match meaning {
+            SparseValidityMeaning::NullPositions => {
+                num_slots.checked_sub(cardinality).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} null cardinality {} exceeds slots {}",
+                            cardinality, num_slots
+                        )
+                        .into(),
+                    )
+                })
+            }
+            SparseValidityMeaning::ValidPositions => Ok(cardinality),
+        }
+    }
+
+    fn count_set_decoder(
+        count_set: &pb21::SparseCountSet,
+        cardinality: u64,
+        label: &str,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<SparseCountSetDecoder> {
+        Self::count_buffer_count(count_set, cardinality, label)?;
+        let counts = count_set.counts.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} count set is missing its variant").into(),
+            )
+        })?;
+        Ok(match counts {
+            pb21::sparse_count_set::Counts::Empty(_) => SparseCountSetDecoder::Empty,
+            pb21::sparse_count_set::Counts::Constant(constant) => SparseCountSetDecoder::Constant {
+                value: constant.value,
+                len: cardinality,
+            },
+            pb21::sparse_count_set::Counts::Explicit(compression) => {
+                SparseCountSetDecoder::Explicit {
+                    decompressor: Self::create_position_decompressor(
+                        compression,
+                        label,
+                        decompressors,
+                    )?,
+                    encoding: compression.clone(),
+                    count: cardinality,
+                }
+            }
+        })
+    }
+
+    fn add_buffer_count(count: &mut usize, additional: usize) -> Result<()> {
+        *count = count.checked_add(additional).ok_or_else(|| {
+            Error::invalid_input_source("Sparse structural buffer count overflows".into())
+        })?;
+        Ok(())
+    }
+
+    fn structural_buffer_count(layers: &[pb21::SparseStructuralLayer]) -> Result<usize> {
+        layers
+            .iter()
+            .try_fold(0_usize, |mut count, layer| -> Result<usize> {
+                match Self::require_layer(layer)? {
+                    pb21::sparse_structural_layer::Layer::Validity(layer) => {
+                        let (validity_buffers, _, _) = Self::validity_buffer_count(
+                            Self::require_validity_set(&layer.validity, "validity")?,
+                            layer.num_slots,
+                            "validity positions",
+                        )?;
+                        Self::add_buffer_count(&mut count, validity_buffers)?;
+                    }
+                    pb21::sparse_structural_layer::Layer::List(layer) => {
+                        let non_empty_positions = Self::require_position_set(
+                            &layer.non_empty_positions,
+                            "list non-empty",
+                        )?;
+                        let num_non_empty = Self::position_cardinality(
+                            non_empty_positions,
+                            layer.num_slots,
+                            "list non-empty positions",
+                        )?;
+                        let non_empty_buffers = Self::position_buffer_count(
+                            non_empty_positions,
+                            layer.num_slots,
+                            "list non-empty positions",
+                        )?;
+                        Self::add_buffer_count(&mut count, non_empty_buffers)?;
+                        let (validity_buffers, validity_meaning, validity_cardinality) =
+                            Self::validity_buffer_count(
+                                Self::require_validity_set(&layer.validity, "list")?,
+                                layer.num_slots,
+                                "list validity positions",
+                            )?;
+                        Self::add_buffer_count(&mut count, validity_buffers)?;
+                        let num_valid_slots = Self::num_valid_slots(
+                            validity_meaning,
+                            validity_cardinality,
+                            layer.num_slots,
+                            "list validity",
+                        )?;
+                        if num_non_empty > num_valid_slots {
+                            return Err(Error::invalid_input_source(
+                                format!(
+                                    "Sparse structural list has {} non-empty slots but only {} valid slots",
+                                    num_non_empty, num_valid_slots
+                                )
+                                .into(),
+                            ));
+                        }
+                        let counts = Self::require_count_set(&layer.counts, "list counts")?;
+                        let count_buffers =
+                            Self::count_buffer_count(counts, num_non_empty, "list counts")?;
+                        Self::add_buffer_count(&mut count, count_buffers)?;
+                        if let Some(child_slots) =
+                            Self::count_set_child_slots(counts, num_non_empty, "list counts")?
+                            && child_slots != layer.num_child_slots
+                        {
+                            return Err(Error::invalid_input_source(
+                                format!(
+                                    "Sparse structural list count sum {} does not match child slots {}",
+                                    child_slots, layer.num_child_slots
+                                )
+                                .into(),
+                            ));
+                        }
+                    }
+                    pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => {
+                        if layer.dimension == 0 {
+                            return Err(Error::invalid_input_source(
+                                "Sparse structural fixed-size-list dimension is zero".into(),
+                            ));
+                        }
+                        layer.num_slots.checked_mul(layer.dimension).ok_or_else(|| {
+                            Error::invalid_input_source(
+                                format!(
+                                    "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}",
+                                    layer.num_slots, layer.dimension
+                                )
+                                .into(),
+                            )
+                        })?;
+                        let (validity_buffers, _, _) = Self::validity_buffer_count(
+                            Self::require_validity_set(&layer.validity, "fixed-size-list")?,
+                            layer.num_slots,
+                            "fixed-size-list validity positions",
+                        )?;
+                        Self::add_buffer_count(&mut count, validity_buffers)?;
+                    }
+                }
+                Ok(count)
+            })
+    }
+
+    fn layer_decompressors(
+        layer: &pb21::SparseStructuralLayer,
+        decompressors: &dyn DecompressionStrategy,
+    ) -> Result<SparseLayerDecompressors> {
+        Ok(match Self::require_layer(layer)? {
+            pb21::sparse_structural_layer::Layer::Validity(layer) => {
+                let (validity, _) = Self::validity_set_decoder(
+                    Self::require_validity_set(&layer.validity, "validity")?,
+                    layer.num_slots,
+                    "validity positions",
+                    decompressors,
+                )?;
+                SparseLayerDecompressors::Validity {
+                    num_slots: layer.num_slots,
+                    validity,
+                }
+            }
+            pb21::sparse_structural_layer::Layer::List(layer) => {
+                let non_empty_positions =
+                    Self::require_position_set(&layer.non_empty_positions, "list non-empty")?;
+                let (non_empty_positions, num_non_empty) = Self::position_set_decoder(
+                    non_empty_positions,
+                    layer.num_slots,
+                    "list non-empty positions",
+                    decompressors,
+                )?;
+                let counts = Self::count_set_decoder(
+                    Self::require_count_set(&layer.counts, "list counts")?,
+                    num_non_empty,
+                    "list counts",
+                    decompressors,
+                )?;
+                let (validity, _) = Self::validity_set_decoder(
+                    Self::require_validity_set(&layer.validity, "list")?,
+                    layer.num_slots,
+                    "list validity positions",
+                    decompressors,
+                )?;
+                SparseLayerDecompressors::List {
+                    num_slots: layer.num_slots,
+                    num_child_slots: layer.num_child_slots,
+                    non_empty_positions,
+                    counts,
+                    validity,
+                }
+            }
+            pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => {
+                let (validity, _) = Self::validity_set_decoder(
+                    Self::require_validity_set(&layer.validity, "fixed-size-list")?,
+                    layer.num_slots,
+                    "fixed-size-list validity positions",
+                    decompressors,
+                )?;
+                SparseLayerDecompressors::FixedSizeList {
+                    num_slots: layer.num_slots,
+                    num_child_slots: layer.num_slots.checked_mul(layer.dimension).ok_or_else(
+                        || {
+                            Error::invalid_input_source(
+                                "Sparse structural fixed-size-list child slot count overflows"
+                                    .into(),
+                            )
+                        },
+                    )?,
+                    dimension: layer.dimension,
+                    validity,
+                }
+            }
+        })
+    }
+
+    fn parse_chunk_meta(&self, meta_bytes: Bytes) -> Result<Vec<ChunkMeta>> {
+        if !meta_bytes.len().is_multiple_of(8) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout metadata length {} is not a multiple of 8",
+                    meta_bytes.len()
+                )
+                .into(),
+            ));
+        }
+
+        let (value_buf_position, value_buf_size) = self.value_buffer()?;
+        let value_buf_end = value_buf_position
+            .checked_add(value_buf_size)
+            .ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout value buffer range overflows".into())
+            })?;
+        let mut rows_counter = 0_u64;
+        let mut offset_bytes = value_buf_position;
+        let mut chunk_meta = Vec::with_capacity(meta_bytes.len() / 8);
+        for chunk in meta_bytes.chunks_exact(8) {
+            let entry: [u8; 8] = chunk.try_into().map_err(|_| {
+                Error::invalid_input_source(
+                    "Sparse layout chunk metadata entry is not 8 bytes".into(),
+                )
+            })?;
+            let divided_bytes_minus_one = u32::from_le_bytes(
+                entry
+                    .get(..4)
+                    .and_then(|bytes| bytes.try_into().ok())
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse layout chunk byte-size field is malformed".into(),
+                        )
+                    })?,
+            );
+            let num_values = u64::from(u32::from_le_bytes(
+                entry
+                    .get(4..)
+                    .and_then(|bytes| bytes.try_into().ok())
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse layout chunk value-count field is malformed".into(),
+                        )
+                    })?,
+            ));
+            if num_values == 0 {
+                return Err(Error::invalid_input_source(
+                    "Sparse layout contains an empty value chunk".into(),
+                ));
+            }
+            if num_values > miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse layout value chunk has {} values, exceeding the mini-block limit {}",
+                        num_values,
+                        miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES
+                    )
+                    .into(),
+                ));
+            }
+            let num_bytes = u64::from(divided_bytes_minus_one)
+                .checked_add(1)
+                .and_then(|units| units.checked_mul(MINIBLOCK_ALIGNMENT as u64))
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse layout value chunk byte size overflows".into(),
+                    )
+                })?;
+            rows_counter = rows_counter.checked_add(num_values).ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout visible item count overflows".into())
+            })?;
+            chunk_meta.push(ChunkMeta {
+                num_values,
+                chunk_size_bytes: num_bytes,
+                offset_bytes,
+            });
+            offset_bytes = offset_bytes.checked_add(num_bytes).ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout value chunk byte range overflows".into())
+            })?;
+        }
+        if rows_counter != self.num_visible_items {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout visible item count mismatch: metadata has {}, layout has {}",
+                    rows_counter, self.num_visible_items
+                )
+                .into(),
+            ));
+        }
+        if offset_bytes != value_buf_end {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout chunk metadata describes {} value bytes, but value buffer has {} bytes",
+                    offset_bytes - value_buf_position,
+                    value_buf_size
+                )
+                .into(),
+            ));
+        }
+        Ok(chunk_meta)
+    }
+
+    fn validate_general_buffer_header(
+        general: &pb21::General,
+        data: &[u8],
+        label: &str,
+    ) -> Result<()> {
+        let compression = general.compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} general compression is missing config").into(),
+            )
+        })?;
+        Self::validate_buffer_compression(Some(compression), label)?;
+        let values = Self::require_encoding(&general.values, label)?;
+        if Self::encoding_contains_general(values) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} contains nested general compression, which is unsupported"
+                )
+                .into(),
+            ));
+        }
+
+        let scheme = invalid_enum(
+            pb21::CompressionScheme::try_from(compression.scheme),
+            "compression scheme",
+        )?;
+        match scheme {
+            pb21::CompressionScheme::CompressionAlgorithmLz4 => {
+                data.get(..4).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} LZ4 buffer is missing its length prefix"
+                        )
+                        .into(),
+                    )
+                })?;
+            }
+            pb21::CompressionScheme::CompressionAlgorithmZstd => {
+                data.get(..8).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural {label} Zstd buffer is missing its length prefix"
+                        )
+                        .into(),
+                    )
+                })?;
+            }
+            pb21::CompressionScheme::CompressionAlgorithmUnspecified => {
+                return Err(Error::invalid_input_source(
+                    format!("Sparse structural {label} general compression scheme is unspecified")
+                        .into(),
+                ));
+            }
+        }
+        Ok(())
+    }
+
+    fn validate_general_child_buffer(
+        encoding: &CompressiveEncoding,
+        data: &[u8],
+        label: &str,
+    ) -> Result<()> {
+        if let Some(pb21::compressive_encoding::Compression::General(general)) =
+            encoding.compression.as_ref()
+        {
+            Self::validate_general_buffer_header(general, data, label)?;
+        }
+        Ok(())
+    }
+
+    fn validate_structural_buffer_headers(
+        encoding: &CompressiveEncoding,
+        data: &[u8],
+        label: &str,
+    ) -> Result<()> {
+        use pb21::compressive_encoding::Compression;
+
+        match encoding.compression.as_ref() {
+            Some(Compression::General(general)) => {
+                Self::validate_general_buffer_header(general, data, label)
+            }
+            Some(Compression::Rle(rle)) => {
+                let values_size = u64::from_le_bytes(
+                    data.get(..8)
+                        .ok_or_else(|| {
+                            Error::invalid_input_source(
+                                format!(
+                                    "Sparse structural {label} RLE buffer is missing its header"
+                                )
+                                .into(),
+                            )
+                        })?
+                        .try_into()
+                        .map_err(|_| {
+                            Error::invalid_input_source(
+                                format!("Sparse structural {label} RLE header is malformed").into(),
+                            )
+                        })?,
+                );
+                let values_size = usize_from_u64(values_size, "RLE values buffer size")?;
+                let values_end = 8_usize.checked_add(values_size).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} RLE values range overflows").into(),
+                    )
+                })?;
+                let values_data = data.get(8..values_end).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} RLE values buffer is truncated").into(),
+                    )
+                })?;
+                let lengths_data = data.get(values_end..).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} RLE run-length buffer is missing")
+                            .into(),
+                    )
+                })?;
+                Self::validate_general_child_buffer(
+                    Self::require_encoding(&rle.values, "RLE values")?,
+                    values_data,
+                    "RLE values",
+                )?;
+                Self::validate_general_child_buffer(
+                    Self::require_encoding(&rle.run_lengths, "RLE run lengths")?,
+                    lengths_data,
+                    "RLE run lengths",
+                )
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn decode_u64_values(
+        decompressor: &dyn BlockDecompressor,
+        encoding: &CompressiveEncoding,
+        data: Bytes,
+        num_values: u64,
+        label: &str,
+    ) -> Result<Vec<u64>> {
+        Self::validate_structural_buffer_headers(encoding, &data, label)?;
+        let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            decompressor.decompress(LanceBuffer::from_bytes(data, 1), num_values)
+        }))
+        .map_err(|_| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} decompression panicked").into(),
+            )
+        })?
+        .map_err(|error| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} decompression failed: {error}").into(),
+            )
+        })?;
+        let fixed = decoded.as_fixed_width().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} did not decode to fixed width data").into(),
+            )
+        })?;
+        if fixed.bits_per_value != 64 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} decoded to {} bits per value, expected 64",
+                    fixed.bits_per_value
+                )
+                .into(),
+            ));
+        }
+        if fixed.num_values != num_values {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} decoded {} values, expected {}",
+                    fixed.num_values, num_values
+                )
+                .into(),
+            ));
+        }
+        let num_values_usize = usize::try_from(num_values).map_err(|_| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} value count exceeds usize::MAX").into(),
+            )
+        })?;
+        let expected_len = num_values_usize
+            .checked_mul(std::mem::size_of::<u64>())
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!("Sparse structural {label} decoded byte length overflows").into(),
+                )
+            })?;
+        if fixed.data.len() != expected_len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} decoded {} bytes, expected {}",
+                    fixed.data.len(),
+                    expected_len
+                )
+                .into(),
+            ));
+        }
+        let values = fixed.data.borrow_to_typed_slice::<u64>();
+        if values.len() != num_values_usize {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} decoded {} u64 values, expected {}",
+                    values.len(),
+                    num_values
+                )
+                .into(),
+            ));
+        }
+        Ok(values.to_vec())
+    }
+
+    fn decode_explicit_positions(
+        decompressor: &Arc<dyn BlockDecompressor>,
+        encoding: &CompressiveEncoding,
+        data: Bytes,
+        num_positions: u64,
+        num_slots: u64,
+        label: &str,
+    ) -> Result<SparsePositionSet> {
+        let deltas =
+            Self::decode_u64_values(decompressor.as_ref(), encoding, data, num_positions, label)?;
+        let mut positions = Vec::with_capacity(deltas.len());
+        let mut current = 0_u64;
+        for (idx, delta) in deltas.into_iter().enumerate() {
+            if idx == 0 {
+                current = delta;
+            } else {
+                if delta == 0 {
+                    return Err(Error::invalid_input_source(
+                        format!("Sparse structural {label} positions must be strictly increasing")
+                            .into(),
+                    ));
+                }
+                current = current.checked_add(delta).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} position overflow").into(),
+                    )
+                })?;
+            }
+            if current >= num_slots {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural {label} position {} is outside layer with {} slots",
+                        current, num_slots
+                    )
+                    .into(),
+                ));
+            }
+            positions.push(current);
+        }
+        SparsePositionSet::from_positions(positions, num_slots, label)
+    }
+
+    fn decode_position_set(
+        decoder: &SparsePositionSetDecoder,
+        buffers: &mut impl Iterator<Item = Bytes>,
+        label: &str,
+    ) -> Result<SparsePositionSet> {
+        match decoder {
+            SparsePositionSetDecoder::Empty => Ok(SparsePositionSet::empty()),
+            SparsePositionSetDecoder::All { len } => Ok(SparsePositionSet::all(*len)),
+            SparsePositionSetDecoder::Range { start, len } => {
+                Ok(SparsePositionSet::range(*start, *len))
+            }
+            SparsePositionSetDecoder::Explicit {
+                decompressor,
+                encoding,
+                count,
+                domain_len,
+            } => Self::decode_explicit_positions(
+                decompressor,
+                encoding,
+                Self::next_structural_buffer(buffers, label)?,
+                *count,
+                *domain_len,
+                label,
+            ),
+        }
+    }
+
+    fn decode_validity_set(
+        decoder: &SparseValiditySetDecoder,
+        buffers: &mut impl Iterator<Item = Bytes>,
+        label: &str,
+    ) -> Result<SparseValiditySet> {
+        Ok(SparseValiditySet {
+            meaning: decoder.meaning,
+            positions: Self::decode_position_set(&decoder.positions, buffers, label)?,
+        })
+    }
+
+    fn decode_count_set(
+        decoder: &SparseCountSetDecoder,
+        buffers: &mut impl Iterator<Item = Bytes>,
+        label: &str,
+    ) -> Result<SparseCountSet> {
+        match decoder {
+            SparseCountSetDecoder::Empty => Ok(SparseCountSet::Empty),
+            SparseCountSetDecoder::Constant { value, len } => {
+                Ok(SparseCountSet::constant(*value, *len))
+            }
+            SparseCountSetDecoder::Explicit {
+                decompressor,
+                encoding,
+                count,
+            } => {
+                let counts = Self::decode_u64_values(
+                    decompressor.as_ref(),
+                    encoding,
+                    Self::next_structural_buffer(buffers, label)?,
+                    *count,
+                    label,
+                )?;
+                SparseCountSet::from_counts(counts)
+            }
+        }
+    }
+
+    fn next_structural_buffer(
+        buffers: &mut impl Iterator<Item = Bytes>,
+        label: &str,
+    ) -> Result<Bytes> {
+        buffers.next().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} is missing its buffer").into(),
+            )
+        })
+    }
+
+    fn decode_layer(
+        layer: &SparseLayerDecompressors,
+        buffers: &mut impl Iterator<Item = Bytes>,
+    ) -> Result<SparseStructuralLayerPlan> {
+        Ok(match layer {
+            SparseLayerDecompressors::Validity {
+                num_slots,
+                validity,
+            } => SparseStructuralLayerPlan::Validity {
+                num_slots: *num_slots,
+                validity: Self::decode_validity_set(validity, buffers, "validity positions")?,
+            },
+            SparseLayerDecompressors::List {
+                num_slots,
+                num_child_slots,
+                non_empty_positions,
+                counts,
+                validity,
+            } => {
+                let non_empty_positions = Self::decode_position_set(
+                    non_empty_positions,
+                    buffers,
+                    "list non-empty positions",
+                )?;
+                let counts = Self::decode_count_set(counts, buffers, "list counts")?;
+                let validity =
+                    Self::decode_validity_set(validity, buffers, "list validity positions")?;
+                let actual_child_slots = counts.sum()?;
+                if actual_child_slots != *num_child_slots {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list count sum {} does not match declared child slots {}",
+                            actual_child_slots, num_child_slots
+                        )
+                        .into(),
+                    ));
+                }
+                SparseStructuralLayerPlan::List {
+                    num_slots: *num_slots,
+                    num_child_slots: *num_child_slots,
+                    non_empty_positions,
+                    counts,
+                    validity,
+                }
+            }
+            SparseLayerDecompressors::FixedSizeList {
+                num_slots,
+                num_child_slots,
+                dimension,
+                validity,
+            } => {
+                let expected_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}",
+                            num_slots, dimension
+                        )
+                        .into(),
+                    )
+                })?;
+                if expected_child_slots != *num_child_slots {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list child slot count {} does not match slots {} * dimension {}",
+                            num_child_slots, num_slots, dimension
+                        )
+                        .into(),
+                    ));
+                }
+                SparseStructuralLayerPlan::FixedSizeList {
+                    num_slots: *num_slots,
+                    dimension: *dimension,
+                    validity: Self::decode_validity_set(
+                        validity,
+                        buffers,
+                        "fixed-size-list validity positions",
+                    )?,
+                }
+            }
+        })
+    }
+
+    fn lookup_value_chunks(&self, chunk_indices: &[usize]) -> Result<Vec<LoadedChunk>> {
+        let page_meta = self.page_meta.as_ref().ok_or_else(|| {
+            Error::internal("Sparse page scheduler has not been initialized".to_string())
+        })?;
+        chunk_indices
+            .iter()
+            .map(|&chunk_idx| {
+                let chunk_meta = page_meta.chunk_meta.get(chunk_idx).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse layout missing value chunk metadata for chunk {chunk_idx}")
+                            .into(),
+                    )
+                })?;
+                let bytes_start = chunk_meta.offset_bytes;
+                let bytes_end = bytes_start
+                    .checked_add(chunk_meta.chunk_size_bytes)
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!(
+                                "Sparse layout value chunk {} byte range overflows",
+                                chunk_idx
+                            )
+                            .into(),
+                        )
+                    })?;
+                Ok(LoadedChunk {
+                    byte_range: bytes_start..bytes_end,
+                    items_in_chunk: chunk_meta.num_values,
+                    chunk_idx,
+                    data: LanceBuffer::empty(),
+                })
+            })
+            .collect()
+    }
+
+    fn value_chunk_index(chunk_value_offsets: &[u64], value: u64) -> Result<usize> {
+        let total_values = chunk_value_offsets.last().copied().ok_or_else(|| {
+            Error::invalid_input_source("Sparse layout has no value chunk offsets".into())
+        })?;
+        if chunk_value_offsets.len() < 2 || value >= total_values {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout value index {} is outside {} visible items",
+                    value, total_values
+                )
+                .into(),
+            ));
+        }
+        chunk_value_offsets
+            .partition_point(|&offset| offset <= value)
+            .checked_sub(1)
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!("Sparse layout value index {value} is before the first chunk").into(),
+                )
+            })
+    }
+
+    fn value_chunk_range(
+        chunk_value_offsets: &[u64],
+        value_range: Range<u64>,
+    ) -> Result<Range<usize>> {
+        if value_range.is_empty() {
+            return Ok(0..0);
+        }
+        let total_values = chunk_value_offsets.last().copied().ok_or_else(|| {
+            Error::invalid_input_source("Sparse layout has no value chunk offsets".into())
+        })?;
+        if value_range.start > value_range.end || value_range.end > total_values {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse layout value range {}..{} is outside {} visible items",
+                    value_range.start, value_range.end, total_values
+                )
+                .into(),
+            ));
+        }
+        let start = Self::value_chunk_index(chunk_value_offsets, value_range.start)?;
+        let end = chunk_value_offsets
+            .partition_point(|&offset| offset < value_range.end)
+            .max(start + 1);
+        Ok(start..end)
+    }
+}
+
+impl StructuralPageScheduler for SparseStructuralScheduler {
+    fn initialize<'a>(
+        &'a mut self,
+        io: &Arc<dyn EncodingsIo>,
+    ) -> BoxFuture<'a, Result<Arc<dyn CachedPageData>>> {
+        let (meta_buf_position, meta_buf_size) = match self.metadata_buffer() {
+            Ok(buffer) => buffer,
+            Err(err) => return std::future::ready(Err(err)).boxed(),
+        };
+        let required_ranges = match (|| -> Result<Vec<Range<u64>>> {
+            let mut required_ranges = Vec::new();
+            required_ranges.push(Self::checked_buffer_range(
+                meta_buf_position,
+                meta_buf_size,
+                "metadata",
+            )?);
+            for (position, size) in self.buffer_offsets_and_sizes.iter().skip(2) {
+                required_ranges.push(Self::checked_buffer_range(*position, *size, "structural")?);
+            }
+            Ok(required_ranges)
+        })() {
+            Ok(ranges) => ranges,
+            Err(err) => return std::future::ready(Err(err)).boxed(),
+        };
+        let io_req = io.submit_request(required_ranges, 0);
+
+        async move {
+            let mut buffers = io_req.await?.into_iter();
+            let meta_bytes = buffers.next().ok_or_else(|| {
+                Error::invalid_input_source("Sparse layout is missing chunk metadata buffer".into())
+            })?;
+
+            let chunk_meta = self.parse_chunk_meta(meta_bytes)?;
+            let mut chunk_value_offsets = Vec::with_capacity(chunk_meta.len() + 1);
+            let mut value_offset = 0_u64;
+            chunk_value_offsets.push(value_offset);
+            for chunk in &chunk_meta {
+                value_offset = value_offset.checked_add(chunk.num_values).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse layout visible item offset overflows".into(),
+                    )
+                })?;
+                chunk_value_offsets.push(value_offset);
+            }
+
+            let layers = self
+                .layer_decompressors
+                .iter()
+                .map(|layer| Self::decode_layer(layer, &mut buffers))
+                .collect::<Result<Vec<_>>>()?;
+            let plan = SparseStructuralPlan {
+                layers,
+                num_items: self.num_items,
+                num_visible_items: self.num_visible_items,
+            };
+            plan.validate(self.row_domain)?;
+            if buffers.next().is_some() {
+                return Err(Error::invalid_input_source(
+                    "Sparse layout has unused structural buffers".into(),
+                ));
+            }
+
+            let page_meta = Arc::new(SparseStructuralCacheableState {
+                chunk_meta,
+                chunk_value_offsets: chunk_value_offsets.into(),
+                plan,
+                row_domain: self.row_domain,
+            });
+            self.page_meta = Some(page_meta.clone());
+            Ok(page_meta as Arc<dyn CachedPageData>)
+        }
+        .boxed()
+    }
+
+    fn load(&mut self, data: &Arc<dyn CachedPageData>) {
+        self.page_meta = data
+            .clone()
+            .as_arc_any()
+            .downcast::<SparseStructuralCacheableState>()
+            .ok();
+    }
+
+    fn schedule_ranges(
+        &self,
+        ranges: &[Range<u64>],
+        io: &Arc<dyn EncodingsIo>,
+    ) -> Result<Vec<PageLoadTask>> {
+        let page_meta = self.page_meta.as_ref().ok_or_else(|| {
+            Error::internal("Sparse page scheduler has not been initialized".to_string())
+        })?;
+        let encoded_row_domain = page_meta
+            .row_domain
+            .checked_mul(self.row_scale)
+            .ok_or_else(|| {
+                Error::invalid_input_source("Sparse structural encoded row domain overflows".into())
+            })?;
+        let num_rows = validate_slice_ranges(ranges, encoded_row_domain, "row")?;
+        let ranges = ranges
+            .iter()
+            .map(|range| {
+                if !range.start.is_multiple_of(self.row_scale)
+                    || !range.end.is_multiple_of(self.row_scale)
+                {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural encoded row range {}..{} is not aligned to fixed-size-list scale {}",
+                            range.start, range.end, self.row_scale
+                        )
+                        .into(),
+                    ));
+                }
+                Ok((range.start / self.row_scale)..(range.end / self.row_scale))
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        let mut chunks_needed = Vec::new();
+        let selection = slice_sparse_plan(&page_meta.plan, &ranges, page_meta.row_domain)?;
+        for value_range in &selection.leaf_ranges {
+            chunks_needed.extend(Self::value_chunk_range(
+                &page_meta.chunk_value_offsets,
+                value_range.clone(),
+            )?);
+        }
+        chunks_needed.sort_unstable();
+        chunks_needed.dedup();
+
+        let mut loaded_chunks = self.lookup_value_chunks(&chunks_needed)?;
+        let chunk_ranges = loaded_chunks
+            .iter()
+            .map(|chunk| chunk.byte_range.clone())
+            .collect::<Vec<_>>();
+        let loaded_chunk_data = io.submit_request(chunk_ranges, self.priority);
+        let ranges = VecDeque::from(ranges);
+        let value_decompressor = self.value_decompressor.clone();
+        let value_encoding = self.value_encoding.clone();
+        let data_type = self.data_type.clone();
+        let page_meta = page_meta.clone();
+        let num_buffers = self.num_buffers;
+        let has_large_chunk = self.has_large_chunk;
+        let row_scale = self.row_scale;
+
+        let res = async move {
+            let loaded_chunk_data = loaded_chunk_data.await?;
+            for (loaded_chunk, chunk_data) in loaded_chunks.iter_mut().zip(loaded_chunk_data) {
+                loaded_chunk.data = LanceBuffer::from_bytes(chunk_data, 1);
+            }
+
+            Ok(Box::new(SparseStructuralDecoder {
+                value_decompressor,
+                value_encoding,
+                data_type,
+                page_meta,
+                loaded_chunks: Arc::new(loaded_chunks),
+                ranges,
+                offset_in_current_range: 0,
+                num_rows,
+                row_scale,
+                num_buffers,
+                has_large_chunk,
+            }) as Box<dyn StructuralPageDecoder>)
+        }
+        .boxed();
+        Ok(vec![PageLoadTask {
+            decoder_fut: res,
+            num_rows,
+        }])
+    }
+}
+
+#[derive(Debug)]
+struct SparseStructuralDecoder {
+    value_decompressor: Arc<dyn MiniBlockDecompressor>,
+    value_encoding: CompressiveEncoding,
+    data_type: DataType,
+    page_meta: Arc<SparseStructuralCacheableState>,
+    loaded_chunks: Arc<Vec<LoadedChunk>>,
+    ranges: VecDeque<Range<u64>>,
+    offset_in_current_range: u64,
+    num_rows: u64,
+    row_scale: u64,
+    num_buffers: u64,
+    has_large_chunk: bool,
+}
+
+impl SparseStructuralDecoder {
+    fn drain_ranges(&mut self, mut rows_desired: u64) -> Result<Vec<Range<u64>>> {
+        if !rows_desired.is_multiple_of(self.row_scale) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse page decoder drain of {} encoded rows is not aligned to fixed-size-list scale {}",
+                    rows_desired, self.row_scale
+                )
+                .into(),
+            ));
+        }
+        rows_desired /= self.row_scale;
+        let mut ranges = Vec::new();
+        while rows_desired > 0 {
+            let range = self.ranges.front().ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse page decoder was asked to drain more rows than were scheduled".into(),
+                )
+            })?;
+            let start = range
+                .start
+                .checked_add(self.offset_in_current_range)
+                .ok_or_else(|| {
+                    Error::invalid_input_source("Sparse page decoder row offset overflows".into())
+                })?;
+            let rows_available = range.end.checked_sub(start).ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse page decoder range offset exceeds the scheduled range".into(),
+                )
+            })?;
+            let rows_to_take = rows_available.min(rows_desired);
+            let end = start.checked_add(rows_to_take).ok_or_else(|| {
+                Error::invalid_input_source("Sparse page decoder row range overflows".into())
+            })?;
+            ranges.push(start..end);
+            rows_desired -= rows_to_take;
+            self.offset_in_current_range += rows_to_take;
+            if self.offset_in_current_range == range.end - range.start {
+                self.offset_in_current_range = 0;
+                self.ranges.pop_front();
+            }
+        }
+        Ok(ranges)
+    }
+}
+
+impl StructuralPageDecoder for SparseStructuralDecoder {
+    fn drain(&mut self, num_rows: u64) -> Result<Box<dyn DecodePageTask>> {
+        Ok(Box::new(DecodeSparseStructuralTask {
+            row_ranges: self.drain_ranges(num_rows)?,
+            value_decompressor: self.value_decompressor.clone(),
+            value_encoding: self.value_encoding.clone(),
+            data_type: self.data_type.clone(),
+            page_meta: self.page_meta.clone(),
+            loaded_chunks: self.loaded_chunks.clone(),
+            num_buffers: self.num_buffers,
+            has_large_chunk: self.has_large_chunk,
+        }))
+    }
+
+    fn num_rows(&self) -> u64 {
+        self.num_rows
+    }
+}
+
+#[derive(Debug)]
+struct DecodeSparseStructuralTask {
+    row_ranges: Vec<Range<u64>>,
+    value_decompressor: Arc<dyn MiniBlockDecompressor>,
+    value_encoding: CompressiveEncoding,
+    data_type: DataType,
+    page_meta: Arc<SparseStructuralCacheableState>,
+    loaded_chunks: Arc<Vec<LoadedChunk>>,
+    num_buffers: u64,
+    has_large_chunk: bool,
+}
+
+impl DecodeSparseStructuralTask {
+    fn read_chunk_size(
+        buf: &[u8],
+        offset: &mut usize,
+        width: usize,
+        chunk_idx: usize,
+    ) -> Result<u32> {
+        let end = offset.checked_add(width).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural value chunk {chunk_idx} size header overflows").into(),
+            )
+        })?;
+        let bytes = buf.get(*offset..end).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural value chunk {chunk_idx} has a truncated size header")
+                    .into(),
+            )
+        })?;
+        let size = match width {
+            2 => u32::from(u16::from_le_bytes(bytes.try_into().map_err(|_| {
+                Error::invalid_input_source(
+                    format!("Sparse structural value chunk {chunk_idx} has a malformed u16 size")
+                        .into(),
+                )
+            })?)),
+            4 => u32::from_le_bytes(bytes.try_into().map_err(|_| {
+                Error::invalid_input_source(
+                    format!("Sparse structural value chunk {chunk_idx} has a malformed u32 size")
+                        .into(),
+                )
+            })?),
+            _ => {
+                return Err(Error::internal(format!(
+                    "Unsupported sparse value chunk size width {width}"
+                )));
+            }
+        };
+        *offset = end;
+        Ok(size)
+    }
+
+    fn expected_fixed_bytes(num_values: u64, bits_per_value: u64, label: &str) -> Result<usize> {
+        let bits = num_values.checked_mul(bits_per_value).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} decoded bit length overflows").into(),
+            )
+        })?;
+        usize_from_u64(bits.div_ceil(8), label)
+    }
+
+    fn validate_fixed_buffer(
+        buffer: &LanceBuffer,
+        num_values: u64,
+        bits_per_value: u64,
+        label: &str,
+    ) -> Result<()> {
+        let expected = Self::expected_fixed_bytes(num_values, bits_per_value, label)?;
+        if buffer.len() != expected {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} buffer has {} bytes, expected {}",
+                    buffer.len(),
+                    expected
+                )
+                .into(),
+            ));
+        }
+        Ok(())
+    }
+
+    fn validate_variable_buffer(
+        buffer: &LanceBuffer,
+        num_values: u64,
+        bits_per_offset: u64,
+    ) -> Result<()> {
+        let width = usize_from_u64(bits_per_offset / 8, "variable offset width")?;
+        let offset_count = num_values.checked_add(1).ok_or_else(|| {
+            Error::invalid_input_source("Sparse variable offset count overflows".into())
+        })?;
+        let table_len = usize_from_u64(offset_count, "variable offset count")?
+            .checked_mul(width)
+            .ok_or_else(|| {
+                Error::invalid_input_source("Sparse variable offset table size overflows".into())
+            })?;
+        if buffer.len() < table_len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse variable buffer has {} bytes, smaller than its {}-byte offset table",
+                    buffer.len(),
+                    table_len
+                )
+                .into(),
+            ));
+        }
+        let mut previous = None;
+        for index in 0..usize_from_u64(offset_count, "variable offset count")? {
+            let start = index.checked_mul(width).ok_or_else(|| {
+                Error::invalid_input_source("Sparse variable offset index overflows".into())
+            })?;
+            let end = start.checked_add(width).ok_or_else(|| {
+                Error::invalid_input_source("Sparse variable offset range overflows".into())
+            })?;
+            let bytes = buffer.as_ref().get(start..end).ok_or_else(|| {
+                Error::invalid_input_source("Sparse variable offset table is truncated".into())
+            })?;
+            let offset = match width {
+                4 => u64::from(u32::from_le_bytes(bytes.try_into().map_err(|_| {
+                    Error::invalid_input_source("Sparse variable u32 offset is malformed".into())
+                })?)),
+                8 => u64::from_le_bytes(bytes.try_into().map_err(|_| {
+                    Error::invalid_input_source("Sparse variable u64 offset is malformed".into())
+                })?),
+                _ => {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse variable offset width {} is unsupported",
+                            bits_per_offset
+                        )
+                        .into(),
+                    ));
+                }
+            };
+            if offset < table_len as u64 || offset > buffer.len() as u64 {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse variable offset {} is outside payload range {}..{}",
+                        offset,
+                        table_len,
+                        buffer.len()
+                    )
+                    .into(),
+                ));
+            }
+            if previous.is_some_and(|previous| offset < previous) {
+                return Err(Error::invalid_input_source(
+                    "Sparse variable offsets are not monotonically increasing".into(),
+                ));
+            }
+            previous = Some(offset);
+        }
+        Ok(())
+    }
+
+    fn validate_fsl_buffers(
+        fsl: &pb21::FixedSizeList,
+        buffers: &[LanceBuffer],
+        num_values: u64,
+        buffer_index: &mut usize,
+    ) -> Result<()> {
+        use pb21::compressive_encoding::Compression;
+
+        let child_values = num_values.checked_mul(fsl.items_per_value).ok_or_else(|| {
+            Error::invalid_input_source("Sparse fixed-size-list value count overflows".into())
+        })?;
+        if fsl.has_validity {
+            let validity = buffers.get(*buffer_index).ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse fixed-size-list value validity buffer is missing".into(),
+                )
+            })?;
+            Self::validate_fixed_buffer(validity, child_values, 1, "fixed-size-list validity")?;
+            *buffer_index = buffer_index.checked_add(1).ok_or_else(|| {
+                Error::invalid_input_source("Sparse fixed-size-list buffer index overflows".into())
+            })?;
+        }
+        let values = fsl.values.as_deref().ok_or_else(|| {
+            Error::invalid_input_source("Sparse fixed-size-list value encoding is missing".into())
+        })?;
+        match values.compression.as_ref() {
+            Some(Compression::FixedSizeList(inner)) => {
+                Self::validate_fsl_buffers(inner, buffers, child_values, buffer_index)
+            }
+            Some(Compression::Flat(flat)) => {
+                let values = buffers.get(*buffer_index).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse fixed-size-list leaf value buffer is missing".into(),
+                    )
+                })?;
+                Self::validate_fixed_buffer(
+                    values,
+                    child_values,
+                    flat.bits_per_value,
+                    "fixed-size-list leaf values",
+                )?;
+                *buffer_index = buffer_index.checked_add(1).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse fixed-size-list buffer index overflows".into(),
+                    )
+                })?;
+                Ok(())
+            }
+            _ => Err(Error::invalid_input_source(
+                "Sparse fixed-size-list value encoding is malformed".into(),
+            )),
+        }
+    }
+
+    fn validate_value_buffers(&self, buffers: &[LanceBuffer], num_values: u64) -> Result<()> {
+        use pb21::compressive_encoding::Compression;
+
+        let compression = self.value_encoding.compression.as_ref().ok_or_else(|| {
+            Error::invalid_input_source("Sparse value compression is missing".into())
+        })?;
+        match compression {
+            Compression::Flat(flat) => Self::validate_fixed_buffer(
+                buffers.first().ok_or_else(|| {
+                    Error::invalid_input_source("Sparse flat value buffer is missing".into())
+                })?,
+                num_values,
+                flat.bits_per_value,
+                "flat values",
+            ),
+            Compression::InlineBitpacking(bitpacking) => {
+                let buffer = buffers.first().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse inline-bitpacked value buffer is missing".into(),
+                    )
+                })?;
+                if num_values > 1024 {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse inline-bitpacked chunk has {} values, exceeding 1024",
+                            num_values
+                        )
+                        .into(),
+                    ));
+                }
+                let word_bytes = usize_from_u64(
+                    bitpacking.uncompressed_bits_per_value / 8,
+                    "inline bitpacking word width",
+                )?;
+                let header = buffer.as_ref().get(..word_bytes).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse inline-bitpacked buffer is missing its bit-width header".into(),
+                    )
+                })?;
+                let bit_width =
+                    header
+                        .iter()
+                        .enumerate()
+                        .try_fold(0_u64, |value, (idx, byte)| {
+                            let shift = u32::try_from(idx.checked_mul(8).ok_or_else(|| {
+                                Error::invalid_input_source(
+                                    "Sparse inline bit-width shift overflows".into(),
+                                )
+                            })?)
+                            .map_err(|_| {
+                                Error::invalid_input_source(
+                                    "Sparse inline bit-width shift exceeds u32".into(),
+                                )
+                            })?;
+                            Ok::<_, Error>(value | (u64::from(*byte) << shift))
+                        })?;
+                if bit_width > bitpacking.uncompressed_bits_per_value {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse inline bit width {} exceeds uncompressed width {}",
+                            bit_width, bitpacking.uncompressed_bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                let payload_bytes = usize_from_u64(
+                    bit_width.checked_mul(1024).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse inline-bitpacked payload size overflows".into(),
+                        )
+                    })? / 8,
+                    "inline-bitpacked payload size",
+                )?;
+                let expected = word_bytes.checked_add(payload_bytes).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse inline-bitpacked buffer size overflows".into(),
+                    )
+                })?;
+                if buffer.len() != expected {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse inline-bitpacked buffer has {} bytes, expected {}",
+                            buffer.len(),
+                            expected
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(())
+            }
+            Compression::Variable(variable) => {
+                let offsets = variable
+                    .offsets
+                    .as_deref()
+                    .and_then(|encoding| encoding.compression.as_ref())
+                    .and_then(|compression| match compression {
+                        Compression::Flat(flat) => Some(flat),
+                        _ => None,
+                    })
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse variable offset encoding is malformed".into(),
+                        )
+                    })?;
+                Self::validate_variable_buffer(
+                    buffers.first().ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse variable value buffer is missing".into(),
+                        )
+                    })?,
+                    num_values,
+                    offsets.bits_per_value,
+                )
+            }
+            Compression::Fsst(fsst) => {
+                let variable = fsst
+                    .values
+                    .as_deref()
+                    .and_then(|encoding| encoding.compression.as_ref())
+                    .and_then(|compression| match compression {
+                        Compression::Variable(variable) => Some(variable),
+                        _ => None,
+                    })
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse FSST value encoding is malformed".into(),
+                        )
+                    })?;
+                let offsets = variable
+                    .offsets
+                    .as_deref()
+                    .and_then(|encoding| encoding.compression.as_ref())
+                    .and_then(|compression| match compression {
+                        Compression::Flat(flat) => Some(flat),
+                        _ => None,
+                    })
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse FSST offset encoding is malformed".into(),
+                        )
+                    })?;
+                Self::validate_variable_buffer(
+                    buffers.first().ok_or_else(|| {
+                        Error::invalid_input_source("Sparse FSST value buffer is missing".into())
+                    })?,
+                    num_values,
+                    offsets.bits_per_value,
+                )
+            }
+            Compression::ByteStreamSplit(split) => {
+                let bits = split
+                    .values
+                    .as_deref()
+                    .and_then(|encoding| encoding.compression.as_ref())
+                    .and_then(|compression| match compression {
+                        Compression::Flat(flat) => Some(flat.bits_per_value),
+                        _ => None,
+                    })
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse byte-stream-split encoding is malformed".into(),
+                        )
+                    })?;
+                Self::validate_fixed_buffer(
+                    buffers.first().ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse byte-stream-split buffer is missing".into(),
+                        )
+                    })?,
+                    num_values,
+                    bits,
+                    "byte-stream-split values",
+                )
+            }
+            Compression::FixedSizeList(fsl) => {
+                let mut buffer_index = 0;
+                Self::validate_fsl_buffers(fsl, buffers, num_values, &mut buffer_index)?;
+                if buffer_index != buffers.len() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse fixed-size-list descriptor consumed {} of {} buffers",
+                            buffer_index,
+                            buffers.len()
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(())
+            }
+            Compression::PackedStruct(packed) => {
+                let bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| {
+                    sum.checked_add(*bits).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse packed-struct bit width sum overflows".into(),
+                        )
+                    })
+                })?;
+                Self::validate_fixed_buffer(
+                    buffers.first().ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse packed-struct value buffer is missing".into(),
+                        )
+                    })?,
+                    num_values,
+                    bits,
+                    "packed-struct values",
+                )
+            }
+            Compression::Rle(rle) => {
+                if buffers.len() != 2 {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse RLE value chunk has {} buffers, expected 2",
+                            buffers.len()
+                        )
+                        .into(),
+                    ));
+                }
+                SparseStructuralScheduler::validate_general_child_buffer(
+                    SparseStructuralScheduler::require_encoding(&rle.values, "RLE values")?,
+                    buffers
+                        .first()
+                        .ok_or_else(|| {
+                            Error::invalid_input_source(
+                                "Sparse RLE value buffer is missing after count validation".into(),
+                            )
+                        })?
+                        .as_ref(),
+                    "value chunk RLE values",
+                )?;
+                SparseStructuralScheduler::validate_general_child_buffer(
+                    SparseStructuralScheduler::require_encoding(
+                        &rle.run_lengths,
+                        "RLE run lengths",
+                    )?,
+                    buffers
+                        .get(1)
+                        .ok_or_else(|| {
+                            Error::invalid_input_source(
+                                "Sparse RLE run-length buffer is missing after count validation"
+                                    .into(),
+                            )
+                        })?
+                        .as_ref(),
+                    "value chunk RLE run lengths",
+                )
+            }
+            Compression::General(general) => {
+                let buffer = buffers.first().ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse general-compressed value chunk is missing its first buffer".into(),
+                    )
+                })?;
+                SparseStructuralScheduler::validate_general_buffer_header(
+                    general,
+                    buffer.as_ref(),
+                    "value chunk",
+                )
+            }
+            _ => Err(Error::invalid_input_source(
+                "Sparse value chunk uses an unsupported compression descriptor".into(),
+            )),
+        }
+    }
+
+    fn loaded_chunk(&self, chunk_idx: usize) -> Result<&LoadedChunk> {
+        let index = self
+            .loaded_chunks
+            .binary_search_by_key(&chunk_idx, |chunk| chunk.chunk_idx)
+            .map_err(|_| {
+                Error::internal(format!(
+                    "Sparse structural decode missing loaded value chunk {}",
+                    chunk_idx
+                ))
+            })?;
+        self.loaded_chunks.get(index).ok_or_else(|| {
+            Error::internal(format!(
+                "Sparse structural loaded chunk index {} is missing",
+                index
+            ))
+        })
+    }
+
+    fn decode_value_chunk(&self, chunk: &LoadedChunk) -> Result<DataBlock> {
+        let buf = &chunk.data;
+        if buf.len() < 2 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} is too small for its header: {} bytes",
+                    chunk.chunk_idx,
+                    buf.len()
+                )
+                .into(),
+            ));
+        }
+        let num_levels = u16::from_le_bytes(
+            buf.as_ref()
+                .get(..2)
+                .and_then(|bytes| bytes.try_into().ok())
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural value chunk {} has a malformed level header",
+                            chunk.chunk_idx
+                        )
+                        .into(),
+                    )
+                })?,
+        );
+        let mut offset: usize = 2;
+        if num_levels != 0 {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk unexpectedly contains {} rep/def levels",
+                    num_levels
+                )
+                .into(),
+            ));
+        }
+
+        let size_width = if self.has_large_chunk { 4 } else { 2 };
+        let num_buffers = usize::try_from(self.num_buffers).map_err(|_| {
+            Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk has too many buffers: {}",
+                    self.num_buffers
+                )
+                .into(),
+            )
+        })?;
+        let sizes_len = num_buffers.checked_mul(size_width).ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse structural value chunk buffer-size header overflows".into(),
+            )
+        })?;
+        let header_len = offset.checked_add(sizes_len).ok_or_else(|| {
+            Error::invalid_input_source(
+                "Sparse structural value chunk header length overflows".into(),
+            )
+        })?;
+        if buf.len() < header_len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} is too small for {} buffer sizes: {} bytes",
+                    chunk.chunk_idx,
+                    self.num_buffers,
+                    buf.len()
+                )
+                .into(),
+            ));
+        }
+        let buffer_sizes = (0..num_buffers)
+            .map(|_| Self::read_chunk_size(buf, &mut offset, size_width, chunk.chunk_idx))
+            .collect::<Result<Vec<_>>>()?;
+
+        offset = offset
+            .checked_add(pad_bytes::<MINIBLOCK_ALIGNMENT>(offset))
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "Sparse structural value chunk {} padded header overflows",
+                        chunk.chunk_idx
+                    )
+                    .into(),
+                )
+            })?;
+        if offset > buf.len() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} is missing padding after its header",
+                    chunk.chunk_idx
+                )
+                .into(),
+            ));
+        }
+        let buffers = buffer_sizes
+            .into_iter()
+            .map(|buf_size| {
+                let buf_size = buf_size as usize;
+                let end = offset.checked_add(buf_size).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural value chunk {} buffer size overflows",
+                            chunk.chunk_idx
+                        )
+                        .into(),
+                    )
+                })?;
+                if end > buf.len() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural value chunk {} buffer extends past chunk end",
+                            chunk.chunk_idx
+                        )
+                        .into(),
+                    ));
+                }
+                let buffer = buf.slice_with_length(offset, buf_size);
+                offset = end;
+                offset = offset
+                    .checked_add(pad_bytes::<MINIBLOCK_ALIGNMENT>(offset))
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!(
+                                "Sparse structural value chunk {} padded buffer range overflows",
+                                chunk.chunk_idx
+                            )
+                            .into(),
+                        )
+                    })?;
+                if offset > buf.len() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural value chunk {} padding extends past chunk end",
+                            chunk.chunk_idx
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(buffer)
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        if offset != buf.len() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} consumed {} of {} bytes",
+                    chunk.chunk_idx,
+                    offset,
+                    buf.len()
+                )
+                .into(),
+            ));
+        }
+
+        self.validate_value_buffers(&buffers, chunk.items_in_chunk)?;
+
+        let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            self.value_decompressor
+                .decompress(buffers, chunk.items_in_chunk)
+        }))
+        .map_err(|_| {
+            Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} decompression panicked",
+                    chunk.chunk_idx
+                )
+                .into(),
+            )
+        })?
+        .map_err(|error| {
+            Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} decompression failed: {error}",
+                    chunk.chunk_idx
+                )
+                .into(),
+            )
+        })?;
+        if decoded.num_values() != chunk.items_in_chunk {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural value chunk {} decoded {} values, expected {}",
+                    chunk.chunk_idx,
+                    decoded.num_values(),
+                    chunk.items_in_chunk
+                )
+                .into(),
+            ));
+        }
+        Ok(decoded)
+    }
+
+    fn append_value_range(
+        &self,
+        value_range: Range<u64>,
+        data_builder: &mut DataBlockBuilder,
+        chunk_cache: &mut Option<(usize, DataBlock)>,
+    ) -> Result<()> {
+        let mut value_start = value_range.start;
+        while value_start < value_range.end {
+            let chunk_idx = SparseStructuralScheduler::value_chunk_index(
+                &self.page_meta.chunk_value_offsets,
+                value_start,
+            )?;
+            let chunk_value_start = *self
+                .page_meta
+                .chunk_value_offsets
+                .get(chunk_idx)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse value chunk {} has no start offset", chunk_idx).into(),
+                    )
+                })?;
+            let chunk_value_end = *self
+                .page_meta
+                .chunk_value_offsets
+                .get(chunk_idx.checked_add(1).ok_or_else(|| {
+                    Error::invalid_input_source("Sparse value chunk index overflows".into())
+                })?)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse value chunk {} has no end offset", chunk_idx).into(),
+                    )
+                })?;
+            let take_end = value_range.end.min(chunk_value_end);
+            if value_start < chunk_value_start || take_end <= value_start {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse value range {}..{} does not make progress in chunk {} covering {}..{}",
+                        value_range.start,
+                        value_range.end,
+                        chunk_idx,
+                        chunk_value_start,
+                        chunk_value_end
+                    )
+                    .into(),
+                ));
+            }
+
+            if !matches!(chunk_cache, Some((cached_idx, _)) if *cached_idx == chunk_idx) {
+                let chunk = self.loaded_chunk(chunk_idx)?;
+                *chunk_cache = Some((chunk_idx, self.decode_value_chunk(chunk)?));
+            }
+            let values = &chunk_cache
+                .as_ref()
+                .ok_or_else(|| Error::internal("Sparse structural chunk cache is empty"))?
+                .1;
+            data_builder.append(
+                values,
+                value_start - chunk_value_start..take_end - chunk_value_start,
+            );
+            value_start = take_end;
+        }
+        Ok(())
+    }
+
+    fn decode_checked(self) -> Result<DecodedPage> {
+        let selection = slice_sparse_plan(
+            &self.page_meta.plan,
+            &self.row_ranges,
+            self.page_meta.row_domain,
+        )?;
+        let estimated_size_bytes = self
+            .loaded_chunks
+            .iter()
+            .map(|chunk| chunk.data.len())
+            .try_fold(0_usize, |total, len| total.checked_add(len))
+            .and_then(|total| total.checked_mul(2))
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse structural decode size estimate overflows".into(),
+                )
+            })?;
+        let mut data_builder = DataBlockBuilder::with_capacity_estimate(
+            u64::try_from(estimated_size_bytes).map_err(|_| {
+                Error::invalid_input_source("Sparse structural decode size exceeds u64::MAX".into())
+            })?,
+        );
+        let mut chunk_cache: Option<(usize, DataBlock)> = None;
+        let mut appended_values = false;
+        for value_range in &selection.leaf_ranges {
+            self.append_value_range(value_range.clone(), &mut data_builder, &mut chunk_cache)?;
+            appended_values = true;
+        }
+
+        let data = if appended_values {
+            data_builder.finish()
+        } else {
+            DataBlock::from_array(new_empty_array(&self.data_type))
+        };
+        let unraveler = RepDefUnraveler::new_sparse(selection.plan);
+        Ok(DecodedPage {
+            data,
+            repdef: unraveler,
+        })
+    }
+}
+
+impl DecodePageTask for DecodeSparseStructuralTask {
+    fn decode(self: Box<Self>) -> Result<DecodedPage> {
+        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*self).decode_checked()))
+            .map_err(|_| {
+                Error::invalid_input_source(
+                    "Sparse structural page decoding panicked on malformed input".into(),
+                )
+            })?
+    }
+}
+
+struct SparseStructuralSelection {
+    plan: SparseStructuralPlan,
+    leaf_ranges: Vec<Range<u64>>,
+}
+
+struct SparsePositionSelection {
+    positions: SparsePositionSet,
+    ordinal_ranges: Vec<Range<u64>>,
+}
+
+fn validate_slice_ranges(ranges: &[Range<u64>], domain_len: u64, label: &str) -> Result<u64> {
+    let mut total = 0_u64;
+    for range in ranges {
+        if range.start > range.end || range.end > domain_len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} slice {}..{} is outside domain {}",
+                    range.start, range.end, domain_len
+                )
+                .into(),
+            ));
+        }
+        total = total.checked_add(range.end - range.start).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} slice length overflows").into(),
+            )
+        })?;
+    }
+    Ok(total)
+}
+
+fn push_coalesced_range(ranges: &mut Vec<Range<u64>>, range: Range<u64>) {
+    if range.is_empty() {
+        return;
+    }
+    if let Some(last) = ranges.last_mut()
+        && last.end == range.start
+    {
+        last.end = range.end;
+        return;
+    }
+    ranges.push(range);
+}
+
+fn position_segments_to_set(
+    segments: Vec<Range<u64>>,
+    output_domain: u64,
+    label: &str,
+) -> Result<SparsePositionSet> {
+    let segments = coalesce_ranges(segments);
+    if segments.is_empty() {
+        return Ok(SparsePositionSet::empty());
+    }
+    if segments.len() == 1 {
+        let segment = segments.first().ok_or_else(|| {
+            Error::internal("Sparse structural segment unexpectedly missing".to_string())
+        })?;
+        if segment.start == 0 && segment.end == output_domain {
+            return Ok(SparsePositionSet::all(output_domain));
+        }
+        return Ok(SparsePositionSet::range(
+            segment.start,
+            segment.end - segment.start,
+        ));
+    }
+
+    let total_len = segments
+        .iter()
+        .map(|range| range.end - range.start)
+        .try_fold(0_u64, |sum, len| {
+            sum.checked_add(len).ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!("Sparse structural {label} segment length overflows").into(),
+                )
+            })
+        })?;
+    let total_len = usize::try_from(total_len).map_err(|_| {
+        Error::invalid_input_source(
+            format!("Sparse structural {label} segment length exceeds usize::MAX").into(),
+        )
+    })?;
+    let mut positions = Vec::with_capacity(total_len);
+    for segment in segments {
+        positions.extend(segment);
+    }
+    SparsePositionSet::from_positions(positions, output_domain, label)
+}
+
+fn select_position_set(
+    positions: &SparsePositionSet,
+    ranges: &[Range<u64>],
+    domain_len: u64,
+    label: &str,
+) -> Result<SparsePositionSelection> {
+    let output_domain = validate_slice_ranges(ranges, domain_len, label)?;
+    let mut segments = Vec::new();
+    let mut ordinal_ranges = Vec::new();
+    let mut output_base = 0_u64;
+
+    match positions {
+        SparsePositionSet::Empty => {}
+        SparsePositionSet::All { len } => {
+            if *len != domain_len {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural {label} all set length {} does not match domain {}",
+                        len, domain_len
+                    )
+                    .into(),
+                ));
+            }
+            for range in ranges {
+                let range_len = range.end - range.start;
+                let output_end = output_base.checked_add(range_len).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} output domain overflows").into(),
+                    )
+                })?;
+                push_coalesced_range(&mut segments, output_base..output_end);
+                push_coalesced_range(&mut ordinal_ranges, range.clone());
+                output_base = output_end;
+            }
+        }
+        SparsePositionSet::Range { start, len } => {
+            let source_end = start.checked_add(*len).ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!("Sparse structural {label} range overflows").into(),
+                )
+            })?;
+            if source_end > domain_len {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural {label} range {}..{} is outside domain {}",
+                        start, source_end, domain_len
+                    )
+                    .into(),
+                ));
+            }
+            for range in ranges {
+                let intersect_start = range.start.max(*start);
+                let intersect_end = range.end.min(source_end);
+                if intersect_start < intersect_end {
+                    let out_start = output_base
+                        .checked_add(intersect_start - range.start)
+                        .ok_or_else(|| {
+                            Error::invalid_input_source(
+                                format!("Sparse structural {label} output position overflows")
+                                    .into(),
+                            )
+                        })?;
+                    let out_end = output_base
+                        .checked_add(intersect_end - range.start)
+                        .ok_or_else(|| {
+                            Error::invalid_input_source(
+                                format!("Sparse structural {label} output position overflows")
+                                    .into(),
+                            )
+                        })?;
+                    push_coalesced_range(&mut segments, out_start..out_end);
+                    push_coalesced_range(
+                        &mut ordinal_ranges,
+                        intersect_start - *start..intersect_end - *start,
+                    );
+                }
+                output_base = output_base
+                    .checked_add(range.end - range.start)
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!("Sparse structural {label} output domain overflows").into(),
+                        )
+                    })?;
+            }
+        }
+        SparsePositionSet::Explicit(source_positions) => {
+            let mut out_positions = Vec::new();
+            for range in ranges {
+                let idx_start =
+                    source_positions.partition_point(|position| *position < range.start);
+                let idx_end = source_positions.partition_point(|position| *position < range.end);
+                if idx_start < idx_end {
+                    let selected_positions =
+                        source_positions.get(idx_start..idx_end).ok_or_else(|| {
+                            Error::invalid_input_source(
+                                format!(
+                                    "Sparse structural {label} explicit position slice is invalid"
+                                )
+                                .into(),
+                            )
+                        })?;
+                    for position in selected_positions {
+                        out_positions.push(
+                            output_base
+                                .checked_add(*position - range.start)
+                                .ok_or_else(|| {
+                                    Error::invalid_input_source(
+                                        format!(
+                                            "Sparse structural {label} output position overflows"
+                                        )
+                                        .into(),
+                                    )
+                                })?,
+                        );
+                    }
+                    push_coalesced_range(&mut ordinal_ranges, idx_start as u64..idx_end as u64);
+                }
+                output_base = output_base
+                    .checked_add(range.end - range.start)
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!("Sparse structural {label} output domain overflows").into(),
+                        )
+                    })?;
+            }
+            let positions = SparsePositionSet::from_positions(out_positions, output_domain, label)?;
+            return Ok(SparsePositionSelection {
+                positions,
+                ordinal_ranges,
+            });
+        }
+    }
+
+    Ok(SparsePositionSelection {
+        positions: position_segments_to_set(segments, output_domain, label)?,
+        ordinal_ranges,
+    })
+}
+
+fn select_validity_set(
+    validity: &SparseValiditySet,
+    ranges: &[Range<u64>],
+    domain_len: u64,
+    label: &str,
+) -> Result<SparseValiditySet> {
+    Ok(SparseValiditySet {
+        meaning: validity.meaning,
+        positions: select_position_set(&validity.positions, ranges, domain_len, label)?.positions,
+    })
+}
+
+fn offsets_from_counts(counts: &[u64]) -> Result<Vec<u64>> {
+    let mut offsets = Vec::with_capacity(counts.len() + 1);
+    let mut offset = 0_u64;
+    offsets.push(offset);
+    for count in counts {
+        offset = offset.checked_add(*count).ok_or_else(|| {
+            Error::invalid_input_source("Sparse structural list count offsets overflow".into())
+        })?;
+        offsets.push(offset);
+    }
+    Ok(offsets)
+}
+
+fn coalesce_ranges(ranges: Vec<Range<u64>>) -> Vec<Range<u64>> {
+    let mut coalesced: Vec<Range<u64>> = Vec::with_capacity(ranges.len());
+    for range in ranges {
+        if range.is_empty() {
+            continue;
+        }
+        if let Some(last) = coalesced.last_mut()
+            && last.end == range.start
+        {
+            last.end = range.end;
+            continue;
+        }
+        coalesced.push(range);
+    }
+    coalesced
+}
+
+fn slice_list_layer(
+    num_slots: u64,
+    num_child_slots: u64,
+    non_empty_positions: &SparsePositionSet,
+    counts: &SparseCountSet,
+    validity: &SparseValiditySet,
+    ranges: &[Range<u64>],
+) -> Result<(SparseStructuralLayerPlan, Vec<Range<u64>>)> {
+    if non_empty_positions.len() != counts.len() {
+        return Err(Error::invalid_input_source(
+            "Sparse structural list has mismatched non-empty positions and counts".into(),
+        ));
+    }
+    let count_sum = counts.sum()?;
+    if count_sum != num_child_slots {
+        return Err(Error::invalid_input_source(
+            format!(
+                "Sparse structural list count sum {} does not match child slots {}",
+                count_sum, num_child_slots
+            )
+            .into(),
+        ));
+    }
+
+    let non_empty_selection =
+        select_position_set(non_empty_positions, ranges, num_slots, "list non-empty")?;
+    let out_counts = select_count_set(
+        counts,
+        &non_empty_selection.ordinal_ranges,
+        non_empty_selection.positions.len(),
+    )?;
+    let child_ranges =
+        child_ranges_from_counts(counts, num_child_slots, &non_empty_selection.ordinal_ranges)?;
+    let out_validity = select_validity_set(validity, ranges, num_slots, "list validity")?;
+    let out_num_slots = validate_slice_ranges(ranges, num_slots, "list")?;
+    let out_num_child_slots = out_counts.sum()?;
+    Ok((
+        SparseStructuralLayerPlan::List {
+            num_slots: out_num_slots,
+            num_child_slots: out_num_child_slots,
+            non_empty_positions: non_empty_selection.positions,
+            counts: out_counts,
+            validity: out_validity,
+        },
+        child_ranges,
+    ))
+}
+
+fn select_count_set(
+    counts: &SparseCountSet,
+    ordinal_ranges: &[Range<u64>],
+    selected_len: u64,
+) -> Result<SparseCountSet> {
+    match counts {
+        SparseCountSet::Empty => {
+            if selected_len != 0 {
+                return Err(Error::invalid_input_source(
+                    "Sparse structural selected non-empty positions but counts are empty".into(),
+                ));
+            }
+            Ok(SparseCountSet::Empty)
+        }
+        SparseCountSet::Constant { value, len } => {
+            validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?;
+            Ok(SparseCountSet::constant(*value, selected_len))
+        }
+        SparseCountSet::Explicit {
+            counts: source_counts,
+            ..
+        } => {
+            validate_ordinal_ranges(
+                ordinal_ranges,
+                source_counts.len() as u64,
+                "explicit list counts",
+            )?;
+            let selected_len = usize::try_from(selected_len).map_err(|_| {
+                Error::invalid_input_source(
+                    "Sparse structural selected count length exceeds usize::MAX".into(),
+                )
+            })?;
+            let mut out_counts = Vec::with_capacity(selected_len);
+            for range in ordinal_ranges {
+                let start = usize::try_from(range.start).map_err(|_| {
+                    Error::invalid_input_source(
+                        "Sparse structural count ordinal exceeds usize::MAX".into(),
+                    )
+                })?;
+                let end = usize::try_from(range.end).map_err(|_| {
+                    Error::invalid_input_source(
+                        "Sparse structural count ordinal exceeds usize::MAX".into(),
+                    )
+                })?;
+                out_counts.extend_from_slice(source_counts.get(start..end).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse structural explicit count slice is invalid".into(),
+                    )
+                })?);
+            }
+            SparseCountSet::from_counts(out_counts)
+        }
+    }
+}
+
+fn validate_ordinal_ranges(ranges: &[Range<u64>], len: u64, label: &str) -> Result<()> {
+    for range in ranges {
+        if range.start > range.end || range.end > len {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Sparse structural {label} ordinal range {}..{} is outside {} values",
+                    range.start, range.end, len
+                )
+                .into(),
+            ));
+        }
+    }
+    Ok(())
+}
+
+fn child_ranges_from_counts(
+    counts: &SparseCountSet,
+    num_child_slots: u64,
+    ordinal_ranges: &[Range<u64>],
+) -> Result<Vec<Range<u64>>> {
+    if ordinal_ranges.is_empty() {
+        return Ok(Vec::new());
+    }
+    let child_ranges = match counts {
+        SparseCountSet::Empty => {
+            return Err(Error::invalid_input_source(
+                "Sparse structural selected non-empty positions but counts are empty".into(),
+            ));
+        }
+        SparseCountSet::Constant { value, len } => {
+            let expected_child_slots = value.checked_mul(*len).ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse structural list constant count sum overflows child slots".into(),
+                )
+            })?;
+            if expected_child_slots != num_child_slots {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural list constant count sum {} does not match child slots {}",
+                        expected_child_slots, num_child_slots
+                    )
+                    .into(),
+                ));
+            }
+            validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?;
+            ordinal_ranges
+                .iter()
+                .map(|range| {
+                    let start = range.start.checked_mul(*value).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural list child range start overflows".into(),
+                        )
+                    })?;
+                    let end = range.end.checked_mul(*value).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural list child range end overflows".into(),
+                        )
+                    })?;
+                    Ok(start..end)
+                })
+                .collect::<Result<Vec<_>>>()?
+        }
+        SparseCountSet::Explicit {
+            counts,
+            offsets: value_offsets,
+        } => {
+            let last_offset = *value_offsets.last().ok_or_else(|| {
+                Error::invalid_input_source(
+                    "Sparse structural list count offsets are unexpectedly empty".into(),
+                )
+            })?;
+            if last_offset != num_child_slots {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Sparse structural list count sum {} does not match child slots {}",
+                        last_offset, num_child_slots
+                    )
+                    .into(),
+                ));
+            }
+            validate_ordinal_ranges(ordinal_ranges, counts.len() as u64, "explicit list counts")?;
+            ordinal_ranges
+                .iter()
+                .map(|range| {
+                    let start = usize::try_from(range.start).map_err(|_| {
+                        Error::invalid_input_source(
+                            "Sparse structural count ordinal exceeds usize::MAX".into(),
+                        )
+                    })?;
+                    let end = usize::try_from(range.end).map_err(|_| {
+                        Error::invalid_input_source(
+                            "Sparse structural count ordinal exceeds usize::MAX".into(),
+                        )
+                    })?;
+                    let start_offset = *value_offsets.get(start).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural list start offset is missing".into(),
+                        )
+                    })?;
+                    let end_offset = *value_offsets.get(end).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural list end offset is missing".into(),
+                        )
+                    })?;
+                    Ok(start_offset..end_offset)
+                })
+                .collect::<Result<Vec<_>>>()?
+        }
+    };
+    Ok(coalesce_ranges(child_ranges))
+}
+
+fn slice_sparse_plan(
+    plan: &SparseStructuralPlan,
+    row_ranges: &[Range<u64>],
+    row_domain: u64,
+) -> Result<SparseStructuralSelection> {
+    plan.validate(row_domain)?;
+    let mut selected_ranges = row_ranges.to_vec();
+    let mut selected_domain = row_domain;
+    let mut sliced_layers = Vec::with_capacity(plan.layers.len());
+
+    for layer in &plan.layers {
+        match layer {
+            SparseStructuralLayerPlan::Validity {
+                num_slots,
+                validity,
+            } => {
+                if selected_domain != *num_slots {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural validity slice domain {} does not match {} slots",
+                            selected_domain, num_slots
+                        )
+                        .into(),
+                    ));
+                }
+                let out_num_slots =
+                    validate_slice_ranges(&selected_ranges, *num_slots, "validity")?;
+                sliced_layers.push(SparseStructuralLayerPlan::Validity {
+                    num_slots: out_num_slots,
+                    validity: select_validity_set(
+                        validity,
+                        &selected_ranges,
+                        *num_slots,
+                        "validity",
+                    )?,
+                });
+            }
+            SparseStructuralLayerPlan::List {
+                num_slots,
+                num_child_slots,
+                non_empty_positions,
+                counts,
+                validity,
+                ..
+            } => {
+                if selected_domain != *num_slots {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list slice domain {} does not match {} slots",
+                            selected_domain, num_slots
+                        )
+                        .into(),
+                    ));
+                }
+                let (sliced_layer, child_ranges) = slice_list_layer(
+                    *num_slots,
+                    *num_child_slots,
+                    non_empty_positions,
+                    counts,
+                    validity,
+                    &selected_ranges,
+                )?;
+                sliced_layers.push(sliced_layer);
+                selected_ranges = child_ranges;
+                selected_domain = *num_child_slots;
+            }
+            SparseStructuralLayerPlan::FixedSizeList {
+                num_slots,
+                dimension,
+                validity,
+            } => {
+                if selected_domain != *num_slots {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list slice domain {} does not match {} slots",
+                            selected_domain, num_slots
+                        )
+                        .into(),
+                    ));
+                }
+                let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}",
+                            num_slots, dimension
+                        )
+                        .into(),
+                    )
+                })?;
+                let out_num_slots =
+                    validate_slice_ranges(&selected_ranges, *num_slots, "fixed-size-list")?;
+                let child_ranges = selected_ranges
+                    .iter()
+                    .map(|range| {
+                        let start = range.start.checked_mul(*dimension).ok_or_else(|| {
+                            Error::invalid_input_source(
+                                "Sparse structural fixed-size-list child range start overflows"
+                                    .into(),
+                            )
+                        })?;
+                        let end = range.end.checked_mul(*dimension).ok_or_else(|| {
+                            Error::invalid_input_source(
+                                "Sparse structural fixed-size-list child range end overflows"
+                                    .into(),
+                            )
+                        })?;
+                        Ok(start..end)
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                sliced_layers.push(SparseStructuralLayerPlan::FixedSizeList {
+                    num_slots: out_num_slots,
+                    dimension: *dimension,
+                    validity: select_validity_set(
+                        validity,
+                        &selected_ranges,
+                        *num_slots,
+                        "fixed-size-list validity",
+                    )?,
+                });
+                selected_ranges = child_ranges;
+                selected_domain = num_child_slots;
+            }
+        }
+    }
+
+    if selected_domain != plan.num_visible_items {
+        return Err(Error::invalid_input_source(
+            format!(
+                "Sparse structural selected terminal domain {} does not match {} visible items",
+                selected_domain, plan.num_visible_items
+            )
+            .into(),
+        ));
+    }
+    let num_visible_items =
+        validate_slice_ranges(&selected_ranges, plan.num_visible_items, "visible value")?;
+    let num_items = SparseStructuralPlan::expected_num_items(&sliced_layers, num_visible_items)?;
+    Ok(SparseStructuralSelection {
+        plan: SparseStructuralPlan {
+            layers: sliced_layers,
+            num_items,
+            num_visible_items,
+        },
+        leaf_ranges: coalesce_ranges(selected_ranges),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Mutex;
+
+    use crate::{
+        compression::DefaultDecompressionStrategy,
+        encodings::physical::block::{CompressionConfig, CompressionScheme},
+        testing::SimulatedScheduler,
+    };
+
+    use super::*;
+
+    fn position_set(
+        positions: pb21::sparse_position_set::Positions,
+        num_positions: u64,
+    ) -> Option<pb21::SparsePositionSet> {
+        Some(pb21::SparsePositionSet {
+            positions: Some(positions),
+            num_positions,
+        })
+    }
+
+    fn position_empty() -> Option<pb21::SparsePositionSet> {
+        position_set(
+            pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}),
+            0,
+        )
+    }
+
+    fn position_all(num_positions: u64) -> Option<pb21::SparsePositionSet> {
+        position_set(
+            pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}),
+            num_positions,
+        )
+    }
+
+    fn position_explicit(num_positions: u64) -> Option<pb21::SparsePositionSet> {
+        position_set(
+            pb21::sparse_position_set::Positions::Explicit(ProtobufUtils21::flat(64, None)),
+            num_positions,
+        )
+    }
+
+    fn general_lz4(values: CompressiveEncoding) -> CompressiveEncoding {
+        ProtobufUtils21::wrapped(CompressionConfig::new(CompressionScheme::Lz4, None), values)
+            .unwrap()
+    }
+
+    fn validity(
+        meaning: pb21::sparse_validity_set::Meaning,
+        positions: Option<pb21::SparsePositionSet>,
+    ) -> Option<pb21::SparseValiditySet> {
+        Some(pb21::SparseValiditySet {
+            meaning: meaning as i32,
+            positions,
+        })
+    }
+
+    fn null_positions(
+        positions: Option<pb21::SparsePositionSet>,
+    ) -> Option<pb21::SparseValiditySet> {
+        validity(
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions,
+            positions,
+        )
+    }
+
+    fn count_empty() -> Option<pb21::SparseCountSet> {
+        Some(pb21::SparseCountSet {
+            counts: Some(pb21::sparse_count_set::Counts::Empty(
+                pb21::SparseCountEmpty {},
+            )),
+        })
+    }
+
+    fn count_constant(value: u64) -> Option<pb21::SparseCountSet> {
+        Some(pb21::SparseCountSet {
+            counts: Some(pb21::sparse_count_set::Counts::Constant(
+                pb21::SparseCountConstant { value },
+            )),
+        })
+    }
+
+    fn sparse_layout() -> pb21::SparseLayout {
+        pb21::SparseLayout {
+            value_compression: Some(ProtobufUtils21::flat(32, None)),
+            num_buffers: 1,
+            num_items: 1,
+            num_visible_items: 1,
+            has_large_chunk: false,
+            structural_layers: Vec::new(),
+        }
+    }
+
+    fn validity_layer(
+        num_slots: u64,
+        validity: Option<pb21::SparseValiditySet>,
+    ) -> pb21::SparseStructuralLayer {
+        pb21::SparseStructuralLayer {
+            layer: Some(pb21::sparse_structural_layer::Layer::Validity(
+                pb21::SparseValidityLayer {
+                    num_slots,
+                    validity,
+                },
+            )),
+        }
+    }
+
+    fn list_layer(
+        num_slots: u64,
+        num_child_slots: u64,
+        non_empty_positions: Option<pb21::SparsePositionSet>,
+        counts: Option<pb21::SparseCountSet>,
+        validity: Option<pb21::SparseValiditySet>,
+    ) -> pb21::SparseStructuralLayer {
+        pb21::SparseStructuralLayer {
+            layer: Some(pb21::sparse_structural_layer::Layer::List(
+                pb21::SparseListLayer {
+                    num_slots,
+                    num_child_slots,
+                    non_empty_positions,
+                    counts,
+                    validity,
+                },
+            )),
+        }
+    }
+
+    fn fixed_size_list_layer(
+        num_slots: u64,
+        dimension: u64,
+        validity: Option<pb21::SparseValiditySet>,
+    ) -> pb21::SparseStructuralLayer {
+        pb21::SparseStructuralLayer {
+            layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList(
+                pb21::SparseFixedSizeListLayer {
+                    num_slots,
+                    dimension,
+                    validity,
+                },
+            )),
+        }
+    }
+
+    fn assert_invalid_input_contains(err: Error, expected: &str) {
+        let message = err.to_string();
+        assert!(
+            matches!(&err, Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+        assert!(
+            message.contains(expected),
+            "expected error to contain {expected:?}, got {message}"
+        );
+    }
+
+    #[test]
+    fn rejects_missing_layer_variants_and_item_count_mismatches() {
+        let decompressors = DefaultDecompressionStrategy::default();
+
+        let mut layout = sparse_layout();
+        layout
+            .structural_layers
+            .push(pb21::SparseStructuralLayer { layer: None });
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "missing its layer variant");
+
+        let mut layout = sparse_layout();
+        layout.num_items = 2;
+        layout
+            .structural_layers
+            .push(validity_layer(1, null_positions(position_empty())));
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "layers imply 1");
+    }
+
+    #[test]
+    fn accepts_large_structural_descriptors() {
+        let explicit_values = 8_u64 * 1024 * 1024 + 1;
+        let metadata_size = explicit_values * 8;
+        let value_position = metadata_size;
+        let value_size = explicit_values * 16;
+        let structural_position = value_position + value_size;
+        let structural_size = explicit_values * std::mem::size_of::<u64>() as u64;
+        let mut layout = sparse_layout();
+        layout.num_items = explicit_values;
+        layout.num_visible_items = explicit_values;
+        layout.structural_layers.push(validity_layer(
+            explicit_values,
+            null_positions(position_explicit(explicit_values)),
+        ));
+
+        SparseStructuralScheduler::try_new(
+            &[
+                (0, metadata_size),
+                (value_position, value_size),
+                (structural_position, structural_size),
+            ],
+            0,
+            explicit_values,
+            DataType::Int32,
+            &layout,
+            &DefaultDecompressionStrategy::default(),
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn accepts_deep_supported_value_encodings() {
+        let encoding = (0..300).fold(ProtobufUtils21::flat(32, None), |values, _| {
+            ProtobufUtils21::fsl(1, false, values)
+        });
+
+        assert_eq!(
+            SparseStructuralScheduler::validate_value_encoding(&encoding).unwrap(),
+            1
+        );
+    }
+
+    fn null_set(positions: SparsePositionSet) -> SparseValiditySet {
+        SparseValiditySet {
+            meaning: SparseValidityMeaning::NullPositions,
+            positions,
+        }
+    }
+
+    fn valid_set(positions: SparsePositionSet) -> SparseValiditySet {
+        SparseValiditySet {
+            meaning: SparseValidityMeaning::ValidPositions,
+            positions,
+        }
+    }
+
+    #[test]
+    fn semantic_position_and_count_sets_project_without_materializing_ranges() {
+        let ranges = [1..4];
+        assert_eq!(
+            select_position_set(&SparsePositionSet::Empty, &ranges, 5, "empty")
+                .unwrap()
+                .positions,
+            SparsePositionSet::Empty
+        );
+        assert_eq!(
+            select_position_set(&SparsePositionSet::all(5), &ranges, 5, "all")
+                .unwrap()
+                .positions,
+            SparsePositionSet::all(3)
+        );
+        assert_eq!(
+            select_position_set(&SparsePositionSet::range(1, 3), &ranges, 5, "range")
+                .unwrap()
+                .positions,
+            SparsePositionSet::all(3)
+        );
+        assert_eq!(
+            select_position_set(
+                &SparsePositionSet::Explicit(vec![0, 2, 4]),
+                &[0..1, 4..5],
+                5,
+                "explicit",
+            )
+            .unwrap()
+            .positions,
+            SparsePositionSet::all(2)
+        );
+
+        assert_eq!(
+            select_count_set(&SparseCountSet::Empty, &[], 0).unwrap(),
+            SparseCountSet::Empty
+        );
+        assert_eq!(
+            select_count_set(&SparseCountSet::constant(2, 3), &[0..1, 2..3], 2,).unwrap(),
+            SparseCountSet::constant(2, 2)
+        );
+        assert_eq!(
+            select_count_set(
+                &SparseCountSet::from_counts(vec![1, 2, 3]).unwrap(),
+                &[0..1, 2..3],
+                2,
+            )
+            .unwrap(),
+            SparseCountSet::from_counts(vec![1, 3]).unwrap()
+        );
+    }
+
+    #[test]
+    fn validity_polarities_rebuild_the_same_arrow_domain() {
+        let mut null_builder = BooleanBufferBuilder::new(4);
+        null_set(SparsePositionSet::Explicit(vec![1, 3]))
+            .append_to(&mut null_builder, 4)
+            .unwrap();
+        assert_eq!(
+            null_builder.finish().iter().collect::<Vec<_>>(),
+            vec![true, false, true, false]
+        );
+
+        let mut valid_builder = BooleanBufferBuilder::new(4);
+        valid_set(SparsePositionSet::range(1, 2))
+            .append_to(&mut valid_builder, 4)
+            .unwrap();
+        assert_eq!(
+            valid_builder.finish().iter().collect::<Vec<_>>(),
+            vec![false, true, true, false]
+        );
+    }
+
+    #[test]
+    fn schema_layer_mismatches_are_invalid_input() {
+        let mut missing = SparseStructuralUnraveler::new(SparseStructuralPlan {
+            layers: Vec::new(),
+            num_items: 1,
+            num_visible_items: 1,
+        });
+        let mut validity = BooleanBufferBuilder::new(1);
+        let err = missing.unravel_validity(&mut validity).unwrap_err();
+        assert_invalid_input_contains(err, "fewer layers than the Arrow schema");
+
+        let mut fixed_size_list = SparseStructuralUnraveler::new(SparseStructuralPlan {
+            layers: vec![SparseStructuralLayerPlan::FixedSizeList {
+                num_slots: 2,
+                dimension: 2,
+                validity: null_set(SparsePositionSet::empty()),
+            }],
+            num_items: 4,
+            num_visible_items: 4,
+        });
+        let mut validity = BooleanBufferBuilder::new(2);
+        let err = fixed_size_list.unravel_validity(&mut validity).unwrap_err();
+        assert_invalid_input_contains(err, "does not match the Arrow schema");
+
+        let extra = SparseStructuralUnraveler::new(SparseStructuralPlan {
+            layers: vec![SparseStructuralLayerPlan::Validity {
+                num_slots: 1,
+                validity: null_set(SparsePositionSet::empty()),
+            }],
+            num_items: 1,
+            num_visible_items: 1,
+        });
+        let err = extra.ensure_exhausted().unwrap_err();
+        assert_invalid_input_contains(err, "1 unconsumed layer");
+    }
+
+    fn nested_plan() -> SparseStructuralPlan {
+        SparseStructuralPlan {
+            layers: vec![
+                SparseStructuralLayerPlan::Validity {
+                    num_slots: 6,
+                    validity: null_set(SparsePositionSet::Explicit(vec![1, 4])),
+                },
+                SparseStructuralLayerPlan::List {
+                    num_slots: 6,
+                    num_child_slots: 5,
+                    non_empty_positions: SparsePositionSet::Explicit(vec![0, 2, 5]),
+                    counts: SparseCountSet::from_counts(vec![2, 1, 2]).unwrap(),
+                    validity: null_set(SparsePositionSet::Explicit(vec![1, 4])),
+                },
+                SparseStructuralLayerPlan::FixedSizeList {
+                    num_slots: 5,
+                    dimension: 2,
+                    validity: valid_set(SparsePositionSet::range(1, 3)),
+                },
+            ],
+            num_items: 13,
+            num_visible_items: 10,
+        }
+    }
+
+    #[test]
+    fn discontiguous_projection_preserves_outer_to_inner_order() {
+        let selection = slice_sparse_plan(&nested_plan(), &[5..6, 0..1], 6).unwrap();
+        assert_eq!(selection.leaf_ranges, vec![6..10, 0..4]);
+        assert_eq!(selection.plan.num_visible_items, 8);
+        selection.plan.validate(2).unwrap();
+
+        let SparseStructuralLayerPlan::List {
+            non_empty_positions,
+            counts,
+            ..
+        } = &selection.plan.layers[1]
+        else {
+            panic!("expected projected list layer");
+        };
+        assert_eq!(*non_empty_positions, SparsePositionSet::all(2));
+        assert_eq!(*counts, SparseCountSet::constant(2, 2));
+    }
+
+    #[test]
+    fn no_value_projection_keeps_empty_and_null_list_structure() {
+        let selection = slice_sparse_plan(&nested_plan(), &[1..2, 3..4], 6).unwrap();
+        assert!(selection.leaf_ranges.is_empty());
+        assert_eq!(selection.plan.num_visible_items, 0);
+        selection.plan.validate(2).unwrap();
+
+        let SparseStructuralLayerPlan::List {
+            num_slots,
+            num_child_slots,
+            non_empty_positions,
+            counts,
+            validity,
+        } = &selection.plan.layers[1]
+        else {
+            panic!("expected projected list layer");
+        };
+        assert_eq!((*num_slots, *num_child_slots), (2, 0));
+        assert_eq!(*non_empty_positions, SparsePositionSet::Empty);
+        assert_eq!(*counts, SparseCountSet::Empty);
+        assert_eq!(*validity, null_set(SparsePositionSet::range(0, 1)));
+    }
+
+    #[test]
+    fn rejects_missing_value_compression_and_buffer_count_mismatch() {
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut layout = sparse_layout();
+        layout.value_compression = None;
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "missing value compression");
+
+        let mut layout = sparse_layout();
+        layout.num_buffers = 2;
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "declares 2 value buffers");
+    }
+
+    #[test]
+    fn rejects_inconsistent_chunk_count_before_io() {
+        let decompressors = DefaultDecompressionStrategy::default();
+
+        let layout = sparse_layout();
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 16), (16, 8)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "declares 2 chunks for 1 visible items");
+    }
+
+    #[cfg(feature = "lz4")]
+    #[test]
+    fn accepts_large_general_decompression_headers() {
+        let encoding = general_lz4(ProtobufUtils21::flat(64, None));
+        let Some(pb21::compressive_encoding::Compression::General(general)) =
+            encoding.compression.as_ref()
+        else {
+            panic!("expected General compression");
+        };
+        let declared_size = 65_u32 * 1024 * 1024;
+
+        SparseStructuralScheduler::validate_general_buffer_header(
+            general,
+            &declared_size.to_le_bytes(),
+            "test",
+        )
+        .unwrap();
+    }
+
+    #[cfg(feature = "lz4")]
+    #[tokio::test]
+    async fn rejects_malformed_general_structural_buffers_as_invalid_input() {
+        let mut layout = sparse_layout();
+        layout.structural_layers.push(validity_layer(
+            1,
+            null_positions(position_set(
+                pb21::sparse_position_set::Positions::Explicit(general_lz4(ProtobufUtils21::flat(
+                    64, None,
+                ))),
+                1,
+            )),
+        ));
+        let decompressors = DefaultDecompressionStrategy::default();
+
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8), (16, 8)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 8]);
+        data.extend_from_slice(&8_u32.to_le_bytes());
+        data.extend_from_slice(&[0xff; 4]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+        let Err(err) = scheduler.initialize(&io).await else {
+            panic!("expected malformed General buffer to be rejected");
+        };
+        assert_invalid_input_contains(err, "decompression failed");
+    }
+
+    #[cfg(feature = "lz4")]
+    #[tokio::test]
+    async fn rejects_malformed_general_value_buffer() {
+        let mut layout = sparse_layout();
+        layout.value_compression = Some(general_lz4(ProtobufUtils21::flat(32, None)));
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&0_u16.to_le_bytes());
+        data.extend_from_slice(&0_u16.to_le_bytes());
+        data.extend_from_slice(&[0; 4]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+        scheduler.initialize(&io).await.unwrap();
+        let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap();
+        let mut decoder = page_tasks.pop().unwrap().decoder_fut.await.unwrap();
+        let Err(err) = decoder.drain(1).unwrap().decode() else {
+            panic!("expected malformed General value buffer to be rejected");
+        };
+        assert_invalid_input_contains(err, "missing its length prefix");
+    }
+
+    #[test]
+    fn rejects_layer_domain_and_fixed_size_list_mismatches() {
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut layout = sparse_layout();
+        layout.num_items = 2;
+        layout.num_visible_items = 2;
+        layout
+            .structural_layers
+            .push(validity_layer(1, null_positions(position_empty())));
+        layout
+            .structural_layers
+            .push(validity_layer(2, null_positions(position_empty())));
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "layer 1 has 2 slots, expected 1");
+
+        let mut layout = sparse_layout();
+        layout.num_items = 4;
+        layout.num_visible_items = 4;
+        layout.structural_layers.push(fixed_size_list_layer(
+            2,
+            3,
+            null_positions(position_empty()),
+        ));
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            2,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "terminal domain has 6 slots");
+    }
+
+    #[test]
+    fn rejects_invalid_validity_and_list_count_semantics() {
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut layout = sparse_layout();
+        layout.structural_layers.push(validity_layer(
+            1,
+            validity(
+                pb21::sparse_validity_set::Meaning::SparseValidityUnspecified,
+                position_empty(),
+            ),
+        ));
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "meaning is unspecified");
+
+        let mut layout = sparse_layout();
+        layout.num_items = 3;
+        layout.num_visible_items = 3;
+        layout.structural_layers.push(list_layer(
+            2,
+            3,
+            position_all(2),
+            count_constant(2),
+            null_positions(position_empty()),
+        ));
+        let err = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            2,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap_err();
+        assert_invalid_input_contains(err, "count sum 4 does not match child slots 3");
+    }
+
+    #[tokio::test]
+    async fn rejects_unordered_explicit_positions_after_decompression() {
+        let mut layout = sparse_layout();
+        layout.num_items = 4;
+        layout.num_visible_items = 4;
+        layout
+            .structural_layers
+            .push(validity_layer(4, null_positions(position_explicit(2))));
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8), (16, 16)],
+            0,
+            4,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&4_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 8]);
+        data.extend_from_slice(&3_u64.to_le_bytes());
+        data.extend_from_slice(&0_u64.to_le_bytes());
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+
+        let Err(err) = scheduler.initialize(&io).await else {
+            panic!("expected unordered sparse positions to be rejected");
+        };
+        assert_invalid_input_contains(err, "positions must be strictly increasing");
+    }
+
+    #[tokio::test]
+    async fn rejects_chunk_metadata_value_and_byte_sum_mismatches() {
+        let layout = sparse_layout();
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let mut data = Vec::new();
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 8]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+        let Err(err) = scheduler.initialize(&io).await else {
+            panic!("expected chunk byte sum mismatch");
+        };
+        assert_invalid_input_contains(err, "describes 16 value bytes");
+
+        let mut layout = sparse_layout();
+        layout.num_items = 2;
+        layout.num_visible_items = 2;
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8)],
+            0,
+            2,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 8]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+        let Err(err) = scheduler.initialize(&io).await else {
+            panic!("expected chunk value sum mismatch");
+        };
+        assert_invalid_input_contains(err, "metadata has 1, layout has 2");
+    }
+
+    #[tokio::test]
+    async fn rejects_malformed_value_chunk_before_decompression() {
+        let layout = sparse_layout();
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, 8)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&1_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 8]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+        scheduler.initialize(&io).await.unwrap();
+        let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap();
+        let page_task = page_tasks.pop().unwrap();
+        let mut decoder = page_task.decoder_fut.await.unwrap();
+        let decode_task = decoder.drain(1).unwrap();
+        let Err(err) = decode_task.decode() else {
+            panic!("expected malformed value chunk to be rejected");
+        };
+        assert_invalid_input_contains(err, "flat values buffer has 0 bytes, expected 4");
+    }
+
+    #[tokio::test]
+    async fn accepts_value_chunks_larger_than_64_mib() {
+        let chunk_size = 65_u64 * 1024 * 1024;
+        let layout = sparse_layout();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 8), (8, chunk_size)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &DefaultDecompressionStrategy::default(),
+        )
+        .unwrap();
+        let words_minus_one = u32::try_from(chunk_size / MINIBLOCK_ALIGNMENT as u64 - 1).unwrap();
+        let mut metadata = Vec::new();
+        metadata.extend_from_slice(&words_minus_one.to_le_bytes());
+        metadata.extend_from_slice(&1_u32.to_le_bytes());
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(metadata)));
+
+        scheduler.initialize(&io).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn rejects_value_chunks_above_the_miniblock_limit() {
+        let num_values = miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES + 1;
+        let mut layout = sparse_layout();
+        layout.num_items = num_values;
+        layout.num_visible_items = num_values;
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 16), (16, 16)],
+            0,
+            num_values,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let mut data = Vec::new();
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&(num_values as u32).to_le_bytes());
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&0_u32.to_le_bytes());
+        data.extend_from_slice(&[0; 16]);
+        let io: Arc<dyn EncodingsIo> = Arc::new(SimulatedScheduler::new(Bytes::from(data)));
+
+        let Err(err) = scheduler.initialize(&io).await else {
+            panic!("expected oversized value chunk to be rejected");
+        };
+        assert_invalid_input_contains(err, "exceeding the mini-block limit");
+    }
+
+    #[derive(Debug, Clone)]
+    struct RecordingIo {
+        data: Bytes,
+        calls: Arc<Mutex<Vec<Vec<Range<u64>>>>>,
+    }
+
+    impl RecordingIo {
+        fn new(data: Bytes) -> Self {
+            Self {
+                data,
+                calls: Arc::new(Mutex::new(Vec::new())),
+            }
+        }
+    }
+
+    impl EncodingsIo for RecordingIo {
+        fn submit_request(
+            &self,
+            ranges: Vec<Range<u64>>,
+            _priority: u64,
+        ) -> BoxFuture<'static, Result<Vec<Bytes>>> {
+            self.calls.lock().unwrap().push(ranges.clone());
+            let data = self.data.clone();
+            async move {
+                ranges
+                    .into_iter()
+                    .map(|range| {
+                        let start = usize_from_u64(range.start, "test range start")?;
+                        let end = usize_from_u64(range.end, "test range end")?;
+                        if start > end || end > data.len() {
+                            return Err(Error::invalid_input_source(
+                                "Test I/O range is outside fixture data".into(),
+                            ));
+                        }
+                        Ok(data.slice(start..end))
+                    })
+                    .collect()
+            }
+            .boxed()
+        }
+    }
+
+    #[tokio::test]
+    async fn selective_read_requests_only_intersecting_value_chunk() {
+        let mut layout = sparse_layout();
+        layout.num_items = 4;
+        layout.num_visible_items = 4;
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 16), (16, 32)],
+            0,
+            4,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+
+        let mut data = Vec::new();
+        for _ in 0..2 {
+            data.extend_from_slice(&1_u32.to_le_bytes());
+            data.extend_from_slice(&2_u32.to_le_bytes());
+        }
+        for values in [[10_i32, 20], [30, 40]] {
+            data.extend_from_slice(&0_u16.to_le_bytes());
+            data.extend_from_slice(&8_u16.to_le_bytes());
+            data.extend_from_slice(&[0; 4]);
+            for value in values {
+                data.extend_from_slice(&value.to_le_bytes());
+            }
+        }
+
+        let io = Arc::new(RecordingIo::new(Bytes::from(data)));
+        let trait_io: Arc<dyn EncodingsIo> = io.clone();
+        scheduler.initialize(&trait_io).await.unwrap();
+        let mut page_tasks = scheduler.schedule_ranges(&[2..3], &trait_io).unwrap();
+        let page_task = page_tasks.pop().unwrap();
+        let mut decoder = page_task.decoder_fut.await.unwrap();
+        let decoded = decoder.drain(1).unwrap().decode().unwrap();
+        assert_eq!(decoded.data.num_values(), 1);
+
+        let calls = io.calls.lock().unwrap();
+        assert_eq!(calls.as_slice(), &[vec![0..16], vec![32..48]]);
+    }
+
+    #[tokio::test]
+    async fn empty_leaf_selection_rebuilds_offsets_without_value_io() {
+        let mut layout = sparse_layout();
+        layout.num_visible_items = 0;
+        layout.structural_layers.push(list_layer(
+            1,
+            0,
+            position_empty(),
+            count_empty(),
+            null_positions(position_empty()),
+        ));
+        let decompressors = DefaultDecompressionStrategy::default();
+        let mut scheduler = SparseStructuralScheduler::try_new(
+            &[(0, 0), (0, 0)],
+            0,
+            1,
+            DataType::Int32,
+            &layout,
+            &decompressors,
+        )
+        .unwrap();
+        let io = Arc::new(RecordingIo::new(Bytes::new()));
+        let trait_io: Arc<dyn EncodingsIo> = io.clone();
+        scheduler.initialize(&trait_io).await.unwrap();
+        let mut page_tasks = scheduler.schedule_ranges(&[0..1], &trait_io).unwrap();
+        let page_task = page_tasks.pop().unwrap();
+        let mut decoder = page_task.decoder_fut.await.unwrap();
+        let decoded = decoder.drain(1).unwrap().decode().unwrap();
+        assert_eq!(decoded.data.num_values(), 0);
+
+        let mut repdef = CompositeRepDefUnraveler::new(vec![decoded.repdef]);
+        let (offsets, validity) = repdef.unravel_offsets::<i32>().unwrap();
+        assert_eq!(offsets.as_ref(), &[0, 0]);
+        assert!(validity.is_none());
+
+        let calls = io.calls.lock().unwrap();
+        assert_eq!(calls.len(), 2);
+        assert!(calls[1].is_empty(), "value payload must not be requested");
+    }
+}
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs
new file mode 100644
index 00000000000..3d82b4a5ef6
--- /dev/null
+++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs
@@ -0,0 +1,2344 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Sparse structural planning and serialization.
+
+use std::iter;
+
+use arrow_buffer::BooleanBuffer;
+use lance_core::{Error, Result, datatypes::Field, utils::bit::pad_bytes};
+
+use crate::{
+    buffer::LanceBuffer,
+    compression::CompressionStrategy,
+    data::{BlockInfo, DataBlock, FixedWidthDataBlock},
+    decoder::PageEncoding,
+    encoder::EncodedPage,
+    format::pb21::{self, CompressiveEncoding},
+    repdef::{NormalizedStructuralLayer, NormalizedStructuralPlan},
+    statistics::ComputeStat,
+};
+
+use super::{
+    SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan,
+    SparseValidityMeaning, SparseValiditySet,
+};
+use crate::encodings::logical::primitive::{
+    FILL_BYTE, MINIBLOCK_ALIGNMENT, miniblock::MiniBlockCompressed,
+};
+
+#[derive(Clone, Copy, Default)]
+struct PositionSetStats {
+    count: u64,
+    first: u64,
+    last: u64,
+    is_contiguous: bool,
+}
+
+impl PositionSetStats {
+    fn observe(&mut self, position: u64) {
+        if self.count == 0 {
+            self.first = position;
+            self.is_contiguous = true;
+        } else if self.last.checked_add(1) != Some(position) {
+            self.is_contiguous = false;
+        }
+        self.last = position;
+        self.count += 1;
+    }
+
+    fn encoded_cost(&self, domain_len: u64) -> u64 {
+        if self.count == 0 || self.count == domain_len || self.is_contiguous {
+            0
+        } else {
+            self.count
+        }
+    }
+
+    fn to_set(
+        self,
+        validity: &BooleanBuffer,
+        want_valid: bool,
+        domain_len: u64,
+        label: &str,
+    ) -> Result<SparsePositionSet> {
+        if self.count == 0 {
+            return Ok(SparsePositionSet::empty());
+        }
+        if self.count == domain_len {
+            return Ok(SparsePositionSet::all(domain_len));
+        }
+        if self.is_contiguous {
+            return Ok(SparsePositionSet::range(self.first, self.count));
+        }
+
+        let capacity = usize::try_from(self.count).map_err(|_| {
+            Error::invalid_input_source(
+                format!("Sparse structural {label} positions exceed usize::MAX").into(),
+            )
+        })?;
+        let mut positions = Vec::with_capacity(capacity);
+        for (index, is_valid) in validity.iter().enumerate() {
+            if is_valid == want_valid {
+                positions.push(u64::try_from(index).map_err(|_| {
+                    Error::invalid_input_source(
+                        format!("Sparse structural {label} position exceeds u64::MAX").into(),
+                    )
+                })?);
+            }
+        }
+        SparsePositionSet::from_positions(positions, domain_len, label)
+    }
+}
+
+fn usize_to_u64(value: usize, label: &str) -> Result<u64> {
+    u64::try_from(value).map_err(|_| {
+        Error::invalid_input_source(
+            format!("Sparse structural {label} {value} exceeds u64::MAX").into(),
+        )
+    })
+}
+
+fn validity_set(
+    validity: Option<&BooleanBuffer>,
+    num_slots: usize,
+    label: &str,
+) -> Result<SparseValiditySet> {
+    let Some(validity) = validity else {
+        return Ok(SparseValiditySet {
+            meaning: SparseValidityMeaning::NullPositions,
+            positions: SparsePositionSet::empty(),
+        });
+    };
+    if validity.len() != num_slots {
+        return Err(Error::invalid_input_source(
+            format!(
+                "Sparse structural {label} validity length {} does not match {} slots",
+                validity.len(),
+                num_slots
+            )
+            .into(),
+        ));
+    }
+
+    let domain_len = usize_to_u64(num_slots, "validity domain")?;
+    let mut valid_stats = PositionSetStats::default();
+    let mut null_stats = PositionSetStats::default();
+    for (index, is_valid) in validity.iter().enumerate() {
+        let index = usize_to_u64(index, "validity position")?;
+        if is_valid {
+            valid_stats.observe(index);
+        } else {
+            null_stats.observe(index);
+        }
+    }
+
+    if null_stats.count == 0 {
+        return Ok(SparseValiditySet {
+            meaning: SparseValidityMeaning::NullPositions,
+            positions: SparsePositionSet::empty(),
+        });
+    }
+    if valid_stats.count == 0 {
+        return Ok(SparseValiditySet {
+            meaning: SparseValidityMeaning::ValidPositions,
+            positions: SparsePositionSet::empty(),
+        });
+    }
+
+    let valid_cost = valid_stats.encoded_cost(domain_len);
+    let null_cost = null_stats.encoded_cost(domain_len);
+    if valid_cost < null_cost {
+        Ok(SparseValiditySet {
+            meaning: SparseValidityMeaning::ValidPositions,
+            positions: valid_stats.to_set(validity, true, domain_len, label)?,
+        })
+    } else {
+        Ok(SparseValiditySet {
+            meaning: SparseValidityMeaning::NullPositions,
+            positions: null_stats.to_set(validity, false, domain_len, label)?,
+        })
+    }
+}
+
+/// Builds the semantic sparse plan directly from the once-normalized Arrow layers.
+pub(in crate::encodings::logical::primitive) fn plan(
+    normalized: &NormalizedStructuralPlan,
+    num_visible_items: u64,
+) -> Result<SparseStructuralPlan> {
+    let mut layers = Vec::with_capacity(normalized.layers().len());
+    let mut num_items = num_visible_items;
+
+    for layer in normalized.layers() {
+        match layer {
+            NormalizedStructuralLayer::Validity {
+                validity,
+                num_slots,
+            } => {
+                layers.push(SparseStructuralLayerPlan::Validity {
+                    num_slots: usize_to_u64(num_slots, "validity slot count")?,
+                    validity: validity_set(validity, num_slots, "validity")?,
+                });
+            }
+            NormalizedStructuralLayer::FixedSizeList {
+                validity,
+                dimension,
+                num_slots,
+            } => {
+                if dimension == 0 {
+                    return Err(Error::invalid_input_source(
+                        "Sparse structural fixed-size-list dimension is zero".into(),
+                    ));
+                }
+                layers.push(SparseStructuralLayerPlan::FixedSizeList {
+                    num_slots: usize_to_u64(num_slots, "fixed-size-list slot count")?,
+                    dimension: usize_to_u64(dimension, "fixed-size-list dimension")?,
+                    validity: validity_set(validity, num_slots, "fixed-size-list validity")?,
+                });
+            }
+            NormalizedStructuralLayer::List {
+                offsets,
+                validity,
+                num_slots,
+            } => {
+                let expected_offsets = num_slots.checked_add(1).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Sparse structural list offset count overflows".into(),
+                    )
+                })?;
+                if offsets.len() != expected_offsets {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list has {} offsets for {} slots",
+                            offsets.len(),
+                            num_slots
+                        )
+                        .into(),
+                    ));
+                }
+                if offsets.first().copied() != Some(0) {
+                    return Err(Error::invalid_input_source(
+                        "Sparse structural list offsets must start at zero".into(),
+                    ));
+                }
+
+                let mut non_empty_positions = Vec::new();
+                let mut counts = Vec::new();
+                for slot in 0..num_slots {
+                    let start = offsets[slot];
+                    let end = offsets[slot + 1];
+                    let count = end.checked_sub(start).ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!(
+                                "Sparse structural list offsets decrease at slot {slot}: {start}..{end}"
+                            )
+                            .into(),
+                        )
+                    })?;
+                    let is_valid = validity.is_none_or(|validity| validity.value(slot));
+                    if !is_valid && count != 0 {
+                        return Err(Error::invalid_input_source(
+                            format!(
+                                "Sparse structural null list slot {slot} has {count} child slots"
+                            )
+                            .into(),
+                        ));
+                    }
+                    if is_valid && count > 0 {
+                        non_empty_positions.push(usize_to_u64(slot, "list position")?);
+                        counts.push(u64::try_from(count).map_err(|_| {
+                            Error::invalid_input_source(
+                                format!("Sparse structural list count {count} exceeds u64::MAX")
+                                    .into(),
+                            )
+                        })?);
+                    }
+                }
+
+                let num_slots_u64 = usize_to_u64(num_slots, "list slot count")?;
+                let num_non_empty = usize_to_u64(non_empty_positions.len(), "list position count")?;
+                num_items = num_items
+                    .checked_add(num_slots_u64 - num_non_empty)
+                    .ok_or_else(|| {
+                        Error::invalid_input_source(
+                            "Sparse structural item count overflows u64".into(),
+                        )
+                    })?;
+                let num_child_slots = offsets.last().copied().ok_or_else(|| {
+                    Error::invalid_input_source("Sparse structural list has no offsets".into())
+                })?;
+                let num_child_slots = u64::try_from(num_child_slots).map_err(|_| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list child slot count {num_child_slots} is negative"
+                        )
+                        .into(),
+                    )
+                })?;
+                layers.push(SparseStructuralLayerPlan::List {
+                    num_slots: num_slots_u64,
+                    num_child_slots,
+                    non_empty_positions: SparsePositionSet::from_positions(
+                        non_empty_positions,
+                        num_slots_u64,
+                        "list non-empty",
+                    )?,
+                    counts: SparseCountSet::from_counts(counts)?,
+                    validity: validity_set(validity, num_slots, "list validity")?,
+                });
+            }
+        }
+    }
+
+    let row_domain = match layers.first() {
+        Some(SparseStructuralLayerPlan::Validity { num_slots, .. })
+        | Some(SparseStructuralLayerPlan::List { num_slots, .. })
+        | Some(SparseStructuralLayerPlan::FixedSizeList { num_slots, .. }) => *num_slots,
+        None => {
+            return Err(Error::invalid_input_source(
+                "Sparse structural encoding requires at least one Arrow structural layer".into(),
+            ));
+        }
+    };
+    let plan = SparseStructuralPlan {
+        layers,
+        num_items,
+        num_visible_items,
+    };
+    plan.validate(row_domain)?;
+    Ok(plan)
+}
+
+/// ConstantLayout remains the canonical representation when no value payload exists.
+pub(in crate::encodings::logical::primitive) fn uses_constant_layout(
+    plan: &SparseStructuralPlan,
+    field: &Field,
+) -> bool {
+    if plan.num_visible_items == 0 {
+        return true;
+    }
+    if matches!(field.data_type(), arrow_schema::DataType::Struct(fields) if fields.is_empty()) {
+        return true;
+    }
+
+    let Some(layer) = plan.layers.last() else {
+        return false;
+    };
+    let (num_slots, validity) = match layer {
+        SparseStructuralLayerPlan::Validity {
+            num_slots,
+            validity,
+        }
+        | SparseStructuralLayerPlan::List {
+            num_slots,
+            validity,
+            ..
+        }
+        | SparseStructuralLayerPlan::FixedSizeList {
+            num_slots,
+            validity,
+            ..
+        } => (*num_slots, validity),
+    };
+    match validity.meaning {
+        SparseValidityMeaning::NullPositions => validity.positions.len() == num_slots,
+        SparseValidityMeaning::ValidPositions => validity.positions.is_empty(),
+    }
+}
+
+fn supports_fixed_size_list_values(data: &DataBlock) -> bool {
+    match data {
+        DataBlock::FixedWidth(_) => true,
+        DataBlock::FixedSizeList(list) => supports_fixed_size_list_values(list.child.as_ref()),
+        DataBlock::Nullable(nullable) => supports_fixed_size_list_values(nullable.data.as_ref()),
+        _ => false,
+    }
+}
+
+/// Whether the sparse writer can encode this value block without changing its value path.
+pub fn supports_value_block(data: &DataBlock) -> bool {
+    match data {
+        DataBlock::FixedWidth(_) | DataBlock::VariableWidth(_) => true,
+        DataBlock::Struct(data) => !data.has_variable_width_child(),
+        DataBlock::FixedSizeList(data) => supports_fixed_size_list_values(data.child.as_ref()),
+        DataBlock::Empty()
+        | DataBlock::Constant(_)
+        | DataBlock::AllNull(_)
+        | DataBlock::Nullable(_)
+        | DataBlock::Opaque(_)
+        | DataBlock::Dictionary(_) => false,
+    }
+}
+
+struct SparseMiniBlockChunk {
+    buffer_sizes: Vec<u32>,
+    num_values: u32,
+}
+
+struct SparseMiniBlockCompressed {
+    data: Vec<LanceBuffer>,
+    chunks: Vec<SparseMiniBlockChunk>,
+}
+
+struct SerializedValuePage {
+    num_buffers: u64,
+    data: LanceBuffer,
+    metadata: LanceBuffer,
+}
+
+pub struct PreparedSparseValues {
+    num_values: u64,
+    value_compression: CompressiveEncoding,
+    values: SerializedValuePage,
+}
+
+pub enum SparseValueInput {
+    Unprepared(DataBlock),
+    Prepared(PreparedSparseValues),
+}
+
+struct EncodedStructuralPlan {
+    layers: Vec<pb21::SparseStructuralLayer>,
+    buffers: Vec<LanceBuffer>,
+}
+
+fn with_explicit_value_counts(
+    compressed: MiniBlockCompressed,
+) -> Result<SparseMiniBlockCompressed> {
+    let mut values_in_previous_chunks = 0_u64;
+    let mut chunks = Vec::with_capacity(compressed.chunks.len());
+    for chunk in compressed.chunks {
+        let num_values = chunk.num_values(values_in_previous_chunks, compressed.num_values);
+        values_in_previous_chunks = values_in_previous_chunks
+            .checked_add(num_values)
+            .ok_or_else(|| Error::internal("Sparse value count overflows u64".to_string()))?;
+        chunks.push(SparseMiniBlockChunk {
+            buffer_sizes: chunk.buffer_sizes,
+            num_values: u32::try_from(num_values).map_err(|_| {
+                Error::invalid_input_source(
+                    format!(
+                        "Sparse value chunk has {num_values} visible values, which exceeds the u32 metadata limit"
+                    )
+                    .into(),
+                )
+            })?,
+        });
+    }
+    if values_in_previous_chunks != compressed.num_values {
+        return Err(Error::internal(format!(
+            "Sparse value chunks describe {values_in_previous_chunks} values, expected {}",
+            compressed.num_values
+        )));
+    }
+    Ok(SparseMiniBlockCompressed {
+        data: compressed.data,
+        chunks,
+    })
+}
+
+fn serialize_value_chunks(
+    compressed: SparseMiniBlockCompressed,
+    support_large_chunk: bool,
+) -> Result<SerializedValuePage> {
+    let bytes_data = compressed.data.iter().map(LanceBuffer::len).sum::<usize>();
+    let num_buffers = compressed.data.len();
+    let mut data_buffer = Vec::with_capacity(bytes_data + 9 * num_buffers);
+    let mut metadata = Vec::with_capacity(compressed.chunks.len() * 8);
+    let mut buffer_offsets = vec![0_usize; num_buffers];
+
+    for chunk in compressed.chunks {
+        if chunk.buffer_sizes.len() != num_buffers {
+            return Err(Error::internal(format!(
+                "Sparse chunk has {} value buffer sizes, expected {num_buffers}",
+                chunk.buffer_sizes.len()
+            )));
+        }
+
+        let chunk_start = data_buffer.len();
+        debug_assert_eq!(chunk_start % MINIBLOCK_ALIGNMENT, 0);
+        data_buffer.extend_from_slice(&0_u16.to_le_bytes());
+        if support_large_chunk {
+            for buffer_size in &chunk.buffer_sizes {
+                data_buffer.extend_from_slice(&buffer_size.to_le_bytes());
+            }
+        } else {
+            for buffer_size in &chunk.buffer_sizes {
+                let buffer_size = u16::try_from(*buffer_size).map_err(|_| {
+                    Error::internal(format!(
+                        "Sparse value buffer size ({buffer_size} bytes) exceeds 16-bit metadata"
+                    ))
+                })?;
+                data_buffer.extend_from_slice(&buffer_size.to_le_bytes());
+            }
+        }
+        let add_padding = |buffer: &mut Vec<u8>| {
+            let padding = pad_bytes::<MINIBLOCK_ALIGNMENT>(buffer.len());
+            buffer.extend(iter::repeat_n(FILL_BYTE, padding));
+        };
+        add_padding(&mut data_buffer);
+
+        for (buffer_size, (buffer, buffer_offset)) in chunk
+            .buffer_sizes
+            .iter()
+            .zip(compressed.data.iter().zip(buffer_offsets.iter_mut()))
+        {
+            let start = *buffer_offset;
+            let end = start.checked_add(*buffer_size as usize).ok_or_else(|| {
+                Error::internal("Sparse value buffer range overflows".to_string())
+            })?;
+            let bytes = buffer.as_ref().get(start..end).ok_or_else(|| {
+                Error::internal(format!(
+                    "Sparse value chunk requests bytes {start}..{end} from a {}-byte buffer",
+                    buffer.len()
+                ))
+            })?;
+            *buffer_offset = end;
+            data_buffer.extend_from_slice(bytes);
+            add_padding(&mut data_buffer);
+        }
+
+        let chunk_bytes = data_buffer.len() - chunk_start;
+        if chunk_bytes == 0 || !chunk_bytes.is_multiple_of(MINIBLOCK_ALIGNMENT) {
+            return Err(Error::internal(format!(
+                "Sparse value chunk size {chunk_bytes} is not a positive multiple of {MINIBLOCK_ALIGNMENT}"
+            )));
+        }
+        let words_minus_one = chunk_bytes / MINIBLOCK_ALIGNMENT - 1;
+        metadata.extend_from_slice(
+            &u32::try_from(words_minus_one)
+                .map_err(|_| {
+                    Error::internal(format!(
+                        "Sparse value chunk size {chunk_bytes} exceeds the metadata limit"
+                    ))
+                })?
+                .to_le_bytes(),
+        );
+        metadata.extend_from_slice(&chunk.num_values.to_le_bytes());
+    }
+
+    for (index, (consumed, buffer)) in buffer_offsets
+        .iter()
+        .zip(compressed.data.iter())
+        .enumerate()
+    {
+        if *consumed != buffer.len() {
+            return Err(Error::internal(format!(
+                "Sparse value buffer {index} consumed {consumed} bytes, expected {}",
+                buffer.len()
+            )));
+        }
+    }
+
+    Ok(SerializedValuePage {
+        num_buffers: usize_to_u64(num_buffers, "value buffer count")?,
+        data: LanceBuffer::from(data_buffer),
+        metadata: LanceBuffer::from(metadata),
+    })
+}
+
+pub fn prepare_values(
+    field: &Field,
+    compression_strategy: &dyn CompressionStrategy,
+    data: DataBlock,
+    support_large_chunk: bool,
+) -> Result<PreparedSparseValues> {
+    match &data {
+        DataBlock::AllNull(_) => {
+            return Err(Error::internal(
+                "All-null values must use ConstantLayout".to_string(),
+            ));
+        }
+        DataBlock::Dictionary(_) => {
+            return Err(Error::not_supported_source(
+                "Sparse layout does not support dictionary data blocks".into(),
+            ));
+        }
+        DataBlock::Struct(data) if data.has_variable_width_child() => {
+            return Err(Error::not_supported_source(
+                "Sparse layout does not support variable-width packed struct data blocks".into(),
+            ));
+        }
+        _ => {}
+    }
+
+    let num_values = data.num_values();
+    let compressor = compression_strategy.create_miniblock_compressor(field, &data)?;
+    let (compressed, value_compression) = compressor.compress(data)?;
+    let values =
+        serialize_value_chunks(with_explicit_value_counts(compressed)?, support_large_chunk)?;
+    Ok(PreparedSparseValues {
+        num_values,
+        value_compression,
+        values,
+    })
+}
+
+fn encode_u64_values(
+    values: Vec<u64>,
+    compression_strategy: &dyn CompressionStrategy,
+) -> Result<(LanceBuffer, CompressiveEncoding)> {
+    let num_values = usize_to_u64(values.len(), "u64 value count")?;
+    let mut block = DataBlock::FixedWidth(FixedWidthDataBlock {
+        data: LanceBuffer::reinterpret_vec(values),
+        bits_per_value: 64,
+        num_values,
+        block_info: BlockInfo::new(),
+    });
+    block.compute_stat();
+    let field = Field::new_arrow("", arrow_schema::DataType::UInt64, false)?;
+    let (compressor, encoding) = compression_strategy.create_block_compressor(&field, &block)?;
+    Ok((compressor.compress(block)?, encoding))
+}
+
+fn positions_to_deltas(positions: &[u64], label: &str) -> Result<Vec<u64>> {
+    let mut previous = 0_u64;
+    positions
+        .iter()
+        .copied()
+        .enumerate()
+        .map(|(index, position)| {
+            if index > 0 && position <= previous {
+                return Err(Error::invalid_input_source(
+                    format!("Sparse structural {label} positions must be strictly increasing")
+                        .into(),
+                ));
+            }
+            let delta = if index == 0 {
+                position
+            } else {
+                position - previous
+            };
+            previous = position;
+            Ok(delta)
+        })
+        .collect()
+}
+
+fn encode_position_set(
+    positions: &SparsePositionSet,
+    compression_strategy: &dyn CompressionStrategy,
+    label: &str,
+) -> Result<(Option<LanceBuffer>, pb21::SparsePositionSet)> {
+    let (buffer, positions_pb) = match positions {
+        SparsePositionSet::Empty => (
+            None,
+            pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}),
+        ),
+        SparsePositionSet::All { .. } => (
+            None,
+            pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}),
+        ),
+        SparsePositionSet::Range { start, len } => (
+            None,
+            pb21::sparse_position_set::Positions::Range(pb21::SparsePositionRange {
+                start: *start,
+                length: *len,
+            }),
+        ),
+        SparsePositionSet::Explicit(positions) => {
+            if positions.is_empty() {
+                return Err(Error::internal(format!(
+                    "Sparse structural {label} explicit set is empty"
+                )));
+            }
+            let (buffer, encoding) =
+                encode_u64_values(positions_to_deltas(positions, label)?, compression_strategy)?;
+            (
+                Some(buffer),
+                pb21::sparse_position_set::Positions::Explicit(encoding),
+            )
+        }
+    };
+    Ok((
+        buffer,
+        pb21::SparsePositionSet {
+            positions: Some(positions_pb),
+            num_positions: positions.len(),
+        },
+    ))
+}
+
+fn encode_count_set(
+    counts: &SparseCountSet,
+    compression_strategy: &dyn CompressionStrategy,
+) -> Result<(Option<LanceBuffer>, pb21::SparseCountSet)> {
+    let (buffer, counts_pb) = match counts {
+        SparseCountSet::Empty => (
+            None,
+            pb21::sparse_count_set::Counts::Empty(pb21::SparseCountEmpty {}),
+        ),
+        SparseCountSet::Constant { value, .. } => (
+            None,
+            pb21::sparse_count_set::Counts::Constant(pb21::SparseCountConstant { value: *value }),
+        ),
+        SparseCountSet::Explicit { counts, .. } => {
+            if counts.is_empty() {
+                return Err(Error::internal(
+                    "Sparse structural explicit count set is empty".to_string(),
+                ));
+            }
+            let (buffer, encoding) = encode_u64_values(counts.to_vec(), compression_strategy)?;
+            (
+                Some(buffer),
+                pb21::sparse_count_set::Counts::Explicit(encoding),
+            )
+        }
+    };
+    Ok((
+        buffer,
+        pb21::SparseCountSet {
+            counts: Some(counts_pb),
+        },
+    ))
+}
+
+fn encode_validity_set(
+    validity: &SparseValiditySet,
+    compression_strategy: &dyn CompressionStrategy,
+    label: &str,
+) -> Result<(Option<LanceBuffer>, pb21::SparseValiditySet)> {
+    let (buffer, positions) =
+        encode_position_set(&validity.positions, compression_strategy, label)?;
+    let meaning = match validity.meaning {
+        SparseValidityMeaning::NullPositions => {
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions
+        }
+        SparseValidityMeaning::ValidPositions => {
+            pb21::sparse_validity_set::Meaning::SparseValidityValidPositions
+        }
+    };
+    Ok((
+        buffer,
+        pb21::SparseValiditySet {
+            meaning: meaning as i32,
+            positions: Some(positions),
+        },
+    ))
+}
+
+fn encode_structural_plan(
+    plan: &SparseStructuralPlan,
+    compression_strategy: &dyn CompressionStrategy,
+) -> Result<EncodedStructuralPlan> {
+    let mut layers = Vec::with_capacity(plan.layers.len());
+    let mut buffers = Vec::new();
+
+    for layer in &plan.layers {
+        match layer {
+            SparseStructuralLayerPlan::Validity {
+                num_slots,
+                validity,
+            } => {
+                let (validity_buffer, validity) =
+                    encode_validity_set(validity, compression_strategy, "validity")?;
+                buffers.extend(validity_buffer);
+                layers.push(pb21::SparseStructuralLayer {
+                    layer: Some(pb21::sparse_structural_layer::Layer::Validity(
+                        pb21::SparseValidityLayer {
+                            num_slots: *num_slots,
+                            validity: Some(validity),
+                        },
+                    )),
+                });
+            }
+            SparseStructuralLayerPlan::List {
+                num_slots,
+                num_child_slots,
+                non_empty_positions,
+                counts,
+                validity,
+            } => {
+                if non_empty_positions.len() != counts.len() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Sparse structural list has {} non-empty positions but {} counts",
+                            non_empty_positions.len(),
+                            counts.len()
+                        )
+                        .into(),
+                    ));
+                }
+                let (position_buffer, non_empty_positions) = encode_position_set(
+                    non_empty_positions,
+                    compression_strategy,
+                    "list non-empty",
+                )?;
+                buffers.extend(position_buffer);
+                let (count_buffer, counts) = encode_count_set(counts, compression_strategy)?;
+                buffers.extend(count_buffer);
+                let (validity_buffer, validity) =
+                    encode_validity_set(validity, compression_strategy, "list validity")?;
+                buffers.extend(validity_buffer);
+                layers.push(pb21::SparseStructuralLayer {
+                    layer: Some(pb21::sparse_structural_layer::Layer::List(
+                        pb21::SparseListLayer {
+                            num_slots: *num_slots,
+                            num_child_slots: *num_child_slots,
+                            non_empty_positions: Some(non_empty_positions),
+                            counts: Some(counts),
+                            validity: Some(validity),
+                        },
+                    )),
+                });
+            }
+            SparseStructuralLayerPlan::FixedSizeList {
+                num_slots,
+                dimension,
+                validity,
+            } => {
+                let (validity_buffer, validity) = encode_validity_set(
+                    validity,
+                    compression_strategy,
+                    "fixed-size-list validity",
+                )?;
+                buffers.extend(validity_buffer);
+                num_slots.checked_mul(*dimension).ok_or_else(|| {
+                    Error::invalid_input_source(
+                        format!(
+                            "Sparse structural fixed-size-list child slot count overflows: slots={num_slots}, dimension={dimension}"
+                        )
+                        .into(),
+                    )
+                })?;
+                layers.push(pb21::SparseStructuralLayer {
+                    layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList(
+                        pb21::SparseFixedSizeListLayer {
+                            num_slots: *num_slots,
+                            dimension: *dimension,
+                            validity: Some(validity),
+                        },
+                    )),
+                });
+            }
+        }
+    }
+
+    Ok(EncodedStructuralPlan { layers, buffers })
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(in crate::encodings::logical::primitive) fn encode_page(
+    column_idx: u32,
+    field: &Field,
+    compression_strategy: &dyn CompressionStrategy,
+    values: SparseValueInput,
+    plan: SparseStructuralPlan,
+    row_number: u64,
+    num_rows: u64,
+    support_large_chunk: bool,
+) -> Result<EncodedPage> {
+    let PreparedSparseValues {
+        num_values,
+        value_compression,
+        values,
+    } = match values {
+        SparseValueInput::Unprepared(data) => {
+            prepare_values(field, compression_strategy, data, support_large_chunk)?
+        }
+        SparseValueInput::Prepared(prepared) => prepared,
+    };
+    if plan.num_visible_items != num_values {
+        return Err(Error::internal(format!(
+            "Sparse structural plan has {} visible items but data has {} values",
+            plan.num_visible_items, num_values
+        )));
+    }
+    let structural = encode_structural_plan(&plan, compression_strategy)?;
+    let description = pb21::PageLayout {
+        layout: Some(pb21::page_layout::Layout::SparseLayout(
+            pb21::SparseLayout {
+                value_compression: Some(value_compression),
+                num_buffers: values.num_buffers,
+                num_items: plan.num_items,
+                num_visible_items: plan.num_visible_items,
+                has_large_chunk: support_large_chunk,
+                structural_layers: structural.layers,
+            },
+        )),
+    };
+
+    let mut page_data = Vec::with_capacity(2 + structural.buffers.len());
+    page_data.push(values.metadata);
+    page_data.push(values.data);
+    page_data.extend(structural.buffers);
+    Ok(EncodedPage {
+        data: page_data,
+        description: PageEncoding::Structural(description),
+        num_rows,
+        row_number,
+        column_idx,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{collections::HashMap, sync::Arc};
+
+    use arrow_array::{
+        Array, ArrayRef, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, Int8Array,
+        Int32Array, LargeListArray, ListArray, StringArray, StructArray,
+        builder::{Int32Builder, MapBuilder, StringBuilder},
+        types::Int8Type,
+    };
+    use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer};
+    use arrow_schema::{DataType, Field as ArrowField, Fields};
+
+    use crate::{
+        constants::{
+            PACKED_STRUCT_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY,
+            STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE,
+        },
+        data::FixedSizeListBlock,
+        encoder::{
+            ColumnIndexSequence, EncodingOptions, FieldEncoder, MIN_PAGE_BUFFER_ALIGNMENT,
+            OutOfLineBuffers, default_encoding_strategy,
+        },
+        testing::{TestCases, check_round_trip_encoding_of_data},
+        version::LanceFileVersion,
+    };
+
+    use super::*;
+
+    fn sparse_metadata() -> HashMap<String, String> {
+        HashMap::from([(
+            STRUCTURAL_ENCODING_META_KEY.to_string(),
+            STRUCTURAL_ENCODING_SPARSE.to_string(),
+        )])
+    }
+
+    fn structural_metadata(value: &str) -> HashMap<String, String> {
+        HashMap::from([(STRUCTURAL_ENCODING_META_KEY.to_string(), value.to_string())])
+    }
+
+    fn null_buffer(validity: impl IntoIterator<Item = bool>) -> NullBuffer {
+        NullBuffer::new(BooleanBuffer::from_iter(validity))
+    }
+
+    fn list_i32(offsets: Vec<i32>, validity: Option<Vec<bool>>) -> ArrayRef {
+        let num_values = offsets.last().copied().unwrap_or_default();
+        let values = Arc::new(Int32Array::from_iter_values(0..num_values)) as ArrayRef;
+        Arc::new(
+            ListArray::try_new(
+                Arc::new(ArrowField::new("item", DataType::Int32, true)),
+                OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                values,
+                validity.map(null_buffer),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn sparse_list_values(
+        num_rows: usize,
+        stride: usize,
+        values: ArrayRef,
+        item_field: Arc<ArrowField>,
+    ) -> ArrayRef {
+        let mut offsets = Vec::with_capacity(num_rows + 1);
+        let mut num_values = 0_i32;
+        offsets.push(num_values);
+        for row in 0..num_rows {
+            if (row + 1).is_multiple_of(stride) || row + 1 == num_rows {
+                num_values += 1;
+            }
+            offsets.push(num_values);
+        }
+        assert_eq!(values.len(), num_values as usize);
+        Arc::new(
+            ListArray::try_new(
+                item_field,
+                OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                values,
+                None,
+            )
+            .unwrap(),
+        )
+    }
+
+    fn sparse_i32_list(num_rows: usize, stride: usize) -> ArrayRef {
+        let num_values = num_rows.div_ceil(stride);
+        sparse_list_values(
+            num_rows,
+            stride,
+            Arc::new(Int32Array::from_iter_values(0..num_values as i32)),
+            Arc::new(ArrowField::new("item", DataType::Int32, true)),
+        )
+    }
+
+    fn unsplittable_nested_list(values: ArrayRef, item_field: Arc<ArrowField>) -> ArrayRef {
+        const NUM_INNER_LISTS: usize = 70_000;
+
+        let mut inner_offsets = vec![0_i32; NUM_INNER_LISTS + 1];
+        assert!(!values.is_empty());
+        assert!(values.len() <= NUM_INNER_LISTS);
+        for value_index in 1..=values.len() {
+            inner_offsets[NUM_INNER_LISTS - values.len() + value_index] = value_index as i32;
+        }
+        let inner = Arc::new(
+            ListArray::try_new(
+                item_field,
+                OffsetBuffer::new(ScalarBuffer::from(inner_offsets)),
+                values,
+                None,
+            )
+            .unwrap(),
+        ) as ArrayRef;
+        Arc::new(
+            ListArray::try_new(
+                Arc::new(ArrowField::new("item", inner.data_type().clone(), true)),
+                OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, NUM_INNER_LISTS as i32])),
+                inner,
+                None,
+            )
+            .unwrap(),
+        )
+    }
+
+    fn variable_packed_struct_values(num_values: usize) -> (ArrayRef, Arc<ArrowField>) {
+        let fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]);
+        let values = Arc::new(StructArray::new(
+            fields.clone(),
+            vec![Arc::new(StringArray::from_iter_values(
+                (0..num_values).map(|index| format!("value-{index}")),
+            ))],
+            None,
+        )) as ArrayRef;
+        let item_field = Arc::new(
+            ArrowField::new("item", DataType::Struct(fields), true).with_metadata(HashMap::from([
+                (PACKED_STRUCT_META_KEY.to_string(), "true".to_string()),
+            ])),
+        );
+        (values, item_field)
+    }
+
+    fn dictionary_values(num_values: usize) -> (ArrayRef, Arc<ArrowField>) {
+        let keys = Int8Array::from_iter_values((0..num_values).map(|index| (index % 2) as i8));
+        let values = Arc::new(StringArray::from(vec!["value-0", "value-1"]));
+        let dictionary = Arc::new(DictionaryArray::<Int8Type>::try_new(keys, values).unwrap());
+        let item_field = Arc::new(ArrowField::new(
+            "item",
+            dictionary.data_type().clone(),
+            true,
+        ));
+        (dictionary, item_field)
+    }
+
+    fn large_list_i32(offsets: Vec<i64>, validity: Option<Vec<bool>>) -> ArrayRef {
+        let num_values = offsets.last().copied().unwrap_or_default();
+        let values = Arc::new(Int32Array::from_iter_values(0..num_values as i32)) as ArrayRef;
+        Arc::new(
+            LargeListArray::try_new(
+                Arc::new(ArrowField::new("item", DataType::Int32, true)),
+                OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                values,
+                validity.map(null_buffer),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn map_i32() -> ArrayRef {
+        let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new());
+        builder.keys().append_value("a");
+        builder.values().append_value(1);
+        builder.append(true).unwrap();
+        builder.append(false).unwrap();
+        builder.append(true).unwrap();
+        builder.keys().append_value("b");
+        builder.values().append_null();
+        builder.keys().append_value("c");
+        builder.values().append_value(3);
+        builder.append(true).unwrap();
+        Arc::new(builder.finish())
+    }
+
+    fn fixed_size_list_struct() -> ArrayRef {
+        let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]);
+        let child = Arc::new(StructArray::new(
+            fields.clone(),
+            vec![Arc::new(Int32Array::from(vec![
+                Some(0),
+                Some(1),
+                None,
+                Some(3),
+                Some(4),
+                Some(5),
+                None,
+                Some(7),
+                Some(8),
+                Some(9),
+                Some(10),
+                None,
+            ]))],
+            Some(null_buffer([
+                true, true, false, true, true, true, true, false, true, true, true, true,
+            ])),
+        )) as ArrayRef;
+        Arc::new(
+            FixedSizeListArray::try_new(
+                Arc::new(ArrowField::new("item", DataType::Struct(fields), true)),
+                2,
+                child,
+                Some(null_buffer([true, false, true, true, false, true])),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn list_fixed_size_list_struct() -> ArrayRef {
+        let struct_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]);
+        let structs = Arc::new(StructArray::new(
+            struct_fields.clone(),
+            vec![Arc::new(Int32Array::from(vec![
+                Some(0),
+                None,
+                Some(2),
+                Some(3),
+                None,
+                Some(5),
+            ]))],
+            Some(null_buffer([true, true, false, true, true, true])),
+        )) as ArrayRef;
+        let fixed_size_list = Arc::new(
+            FixedSizeListArray::try_new(
+                Arc::new(ArrowField::new(
+                    "item",
+                    DataType::Struct(struct_fields),
+                    true,
+                )),
+                2,
+                structs,
+                Some(null_buffer([true, false, true])),
+            )
+            .unwrap(),
+        ) as ArrayRef;
+        Arc::new(
+            ListArray::try_new(
+                Arc::new(ArrowField::new(
+                    "item",
+                    fixed_size_list.data_type().clone(),
+                    true,
+                )),
+                OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 1, 1, 3, 3])),
+                fixed_size_list,
+                Some(null_buffer([true, false, true, true])),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn nullable_struct() -> ArrayRef {
+        let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]);
+        Arc::new(StructArray::new(
+            fields,
+            vec![Arc::new(Int32Array::from(vec![
+                Some(10),
+                Some(20),
+                None,
+                Some(40),
+                Some(50),
+            ]))],
+            Some(null_buffer([true, false, true, true, false])),
+        ))
+    }
+
+    fn deeply_nested() -> ArrayRef {
+        let leaf = Arc::new(Int32Array::from(vec![
+            Some(0),
+            None,
+            Some(2),
+            Some(3),
+            None,
+            Some(5),
+            Some(6),
+            Some(7),
+        ])) as ArrayRef;
+        let inner = Arc::new(
+            LargeListArray::try_new(
+                Arc::new(ArrowField::new("item", DataType::Int32, true)),
+                OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, 2, 2, 3, 5, 5, 8])),
+                leaf,
+                Some(null_buffer([true, false, true, true, true, true])),
+            )
+            .unwrap(),
+        ) as ArrayRef;
+        let struct_fields = Fields::from(vec![ArrowField::new(
+            "inner",
+            inner.data_type().clone(),
+            true,
+        )]);
+        let structs = Arc::new(StructArray::new(
+            struct_fields.clone(),
+            vec![inner],
+            Some(null_buffer([true, true, false, true, true, true])),
+        )) as ArrayRef;
+        Arc::new(
+            ListArray::try_new(
+                Arc::new(ArrowField::new(
+                    "item",
+                    DataType::Struct(struct_fields),
+                    true,
+                )),
+                OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 4, 6])),
+                structs,
+                Some(null_buffer([true, false, true, true, true])),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn page_layout(page: &EncodedPage) -> &pb21::page_layout::Layout {
+        let PageEncoding::Structural(layout) = &page.description else {
+            panic!("expected structural page encoding");
+        };
+        layout.layout.as_ref().expect("page layout must be present")
+    }
+
+    fn create_encoder(
+        array: &ArrayRef,
+        version: LanceFileVersion,
+        metadata: HashMap<String, String>,
+    ) -> Result<Box<dyn FieldEncoder>> {
+        let arrow_field =
+            ArrowField::new("values", array.data_type().clone(), true).with_metadata(metadata);
+        let field = Field::try_from(&arrow_field)?;
+        let strategy = default_encoding_strategy(version);
+        let options = EncodingOptions {
+            cache_bytes_per_column: 1,
+            version,
+            ..Default::default()
+        };
+        strategy.create_field_encoder(
+            strategy.as_ref(),
+            &field,
+            &mut ColumnIndexSequence::default(),
+            &options,
+        )
+    }
+
+    async fn encode_pages(
+        array: ArrayRef,
+        version: LanceFileVersion,
+        metadata: HashMap<String, String>,
+    ) -> Result<Vec<EncodedPage>> {
+        encode_chunks(vec![array], version, metadata).await
+    }
+
+    async fn encode_chunks(
+        arrays: Vec<ArrayRef>,
+        version: LanceFileVersion,
+        metadata: HashMap<String, String>,
+    ) -> Result<Vec<EncodedPage>> {
+        let first = arrays
+            .first()
+            .ok_or_else(|| Error::internal("test input has no arrays".to_string()))?;
+        let mut encoder = create_encoder(first, version, metadata)?;
+        let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT);
+        let mut pages = Vec::new();
+        let mut row_number = 0_u64;
+        for array in arrays {
+            let num_rows = array.len() as u64;
+            for task in encoder.maybe_encode(
+                array,
+                &mut external_buffers,
+                crate::repdef::RepDefBuilder::default(),
+                row_number,
+                num_rows,
+            )? {
+                pages.push(task.await?);
+            }
+            row_number += num_rows;
+        }
+        for task in encoder.flush(&mut external_buffers)? {
+            pages.push(task.await?);
+        }
+        for column in encoder.finish(&mut external_buffers).await? {
+            pages.extend(column.final_pages);
+        }
+        Ok(pages)
+    }
+
+    fn sparse_layout(page: &EncodedPage) -> &pb21::SparseLayout {
+        let pb21::page_layout::Layout::SparseLayout(sparse) = page_layout(page) else {
+            panic!("expected SparseLayout, got {:?}", page_layout(page));
+        };
+        sparse
+    }
+
+    fn list_layer(sparse: &pb21::SparseLayout) -> &pb21::SparseListLayer {
+        sparse
+            .structural_layers
+            .iter()
+            .find_map(|layer| match layer.layer.as_ref() {
+                Some(pb21::sparse_structural_layer::Layer::List(layer)) => Some(layer),
+                _ => None,
+            })
+            .expect("expected sparse list layer")
+    }
+
+    fn validity_layer(layer: &pb21::SparseStructuralLayer) -> Option<&pb21::SparseValidityLayer> {
+        match layer.layer.as_ref() {
+            Some(pb21::sparse_structural_layer::Layer::Validity(layer)) => Some(layer),
+            _ => None,
+        }
+    }
+
+    fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> u64 {
+        match layer.layer.as_ref().expect("expected sparse layer variant") {
+            pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots,
+            pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots,
+            pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots,
+        }
+    }
+
+    fn fixed_size_list_dimension(layer: &pb21::SparseStructuralLayer) -> Option<u64> {
+        match layer.layer.as_ref() {
+            Some(pb21::sparse_structural_layer::Layer::FixedSizeList(layer)) => {
+                Some(layer.dimension)
+            }
+            _ => None,
+        }
+    }
+
+    #[test]
+    fn test_sparse_value_block_eligibility() {
+        let fixed = DataBlock::from_array(Int32Array::from_iter_values(0..4));
+        assert!(supports_value_block(&fixed));
+
+        let variable = DataBlock::from_array(StringArray::from(vec!["a", "b"]));
+        assert!(supports_value_block(&variable));
+
+        let nullable = DataBlock::from_array(Int32Array::from(vec![Some(1), None]));
+        assert!(!supports_value_block(&nullable));
+
+        let all_null = DataBlock::from_array(Int32Array::from(vec![None, None]));
+        assert!(!supports_value_block(&all_null));
+
+        let (dictionary, _) = dictionary_values(4);
+        assert!(!supports_value_block(&DataBlock::from_arrays(
+            &[dictionary],
+            4
+        )));
+
+        let fixed_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, false)]);
+        let fixed_struct = StructArray::new(
+            fixed_fields,
+            vec![Arc::new(Int32Array::from_iter_values(0..4))],
+            None,
+        );
+        assert!(supports_value_block(&DataBlock::from_array(fixed_struct)));
+
+        let variable_fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]);
+        let variable_struct = StructArray::new(
+            variable_fields,
+            vec![Arc::new(StringArray::from(vec!["a", "b"]))],
+            None,
+        );
+        assert!(!supports_value_block(&DataBlock::from_array(
+            variable_struct
+        )));
+
+        let fixed_size_list = FixedSizeListArray::try_new(
+            Arc::new(ArrowField::new("item", DataType::Int32, false)),
+            2,
+            Arc::new(Int32Array::from_iter_values(0..4)),
+            None,
+        )
+        .unwrap();
+        assert!(supports_value_block(&DataBlock::from_array(
+            fixed_size_list
+        )));
+
+        let unsupported_fixed_size_list = DataBlock::FixedSizeList(FixedSizeListBlock {
+            child: Box::new(DataBlock::from_array(StringArray::from(vec![
+                "a", "b", "c", "d",
+            ]))),
+            dimension: 2,
+        });
+        assert!(!supports_value_block(&unsupported_fixed_size_list));
+    }
+
+    fn planned_list(
+        offsets: Vec<i32>,
+        list_validity: Option<Vec<bool>>,
+        leaf_validity: Option<Vec<bool>>,
+    ) -> SparseStructuralPlan {
+        let num_values = u64::try_from(*offsets.last().unwrap()).unwrap();
+        let mut builder = crate::repdef::RepDefBuilder::default();
+        assert!(!builder.add_offsets(
+            OffsetBuffer::new(ScalarBuffer::from(offsets)),
+            list_validity.map(null_buffer),
+        ));
+        if let Some(leaf_validity) = leaf_validity {
+            builder.add_validity_bitmap(null_buffer(leaf_validity));
+        } else {
+            builder.add_no_null(num_values as usize);
+        }
+        let normalized = crate::repdef::RepDefBuilder::normalize(vec![builder]);
+        plan(&normalized, num_values).unwrap()
+    }
+
+    fn planned_list_layer(plan: &SparseStructuralPlan) -> &SparseStructuralLayerPlan {
+        plan.layers
+            .iter()
+            .find(|layer| matches!(layer, SparseStructuralLayerPlan::List { .. }))
+            .expect("expected planned list layer")
+    }
+
+    #[test]
+    fn test_semantic_position_and_count_forms() {
+        let empty = planned_list(vec![0, 0, 0, 0], None, None);
+        assert_eq!(empty.num_items, 3);
+        assert_eq!(empty.num_visible_items, 0);
+        assert!(matches!(
+            planned_list_layer(&empty),
+            SparseStructuralLayerPlan::List {
+                non_empty_positions: SparsePositionSet::Empty,
+                counts: SparseCountSet::Empty,
+                ..
+            }
+        ));
+
+        let all = planned_list(vec![0, 2, 4, 6], None, None);
+        assert_eq!(all.num_items, 6);
+        assert!(matches!(
+            planned_list_layer(&all),
+            SparseStructuralLayerPlan::List {
+                non_empty_positions: SparsePositionSet::All { len: 3 },
+                counts: SparseCountSet::Constant { value: 2, len: 3 },
+                ..
+            }
+        ));
+
+        let range = planned_list(vec![0, 2, 4, 4, 4], None, None);
+        assert_eq!(range.num_items, 6);
+        assert!(matches!(
+            planned_list_layer(&range),
+            SparseStructuralLayerPlan::List {
+                non_empty_positions: SparsePositionSet::Range { start: 0, len: 2 },
+                counts: SparseCountSet::Constant { value: 2, len: 2 },
+                ..
+            }
+        ));
+
+        let explicit = planned_list(vec![0, 1, 1, 4, 4, 6], None, None);
+        assert_eq!(explicit.num_items, 8);
+        assert!(matches!(
+            planned_list_layer(&explicit),
+            SparseStructuralLayerPlan::List {
+                non_empty_positions: SparsePositionSet::Explicit(positions),
+                counts: SparseCountSet::Explicit { counts, .. },
+                ..
+            } if positions == &vec![0, 2, 4] && counts.as_ref() == [1, 3, 2]
+        ));
+    }
+
+    #[test]
+    fn test_validity_polarity_uses_semantic_encoded_cost() {
+        let mostly_valid = BooleanBuffer::from_iter([true, false, true, true, false, true]);
+        let validity = validity_set(Some(&mostly_valid), mostly_valid.len(), "test").unwrap();
+        assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions);
+        assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4]));
+
+        let mostly_null = BooleanBuffer::from_iter([false, true, false, false, true, false]);
+        let validity = validity_set(Some(&mostly_null), mostly_null.len(), "test").unwrap();
+        assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions);
+        assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4]));
+
+        let valid_island = BooleanBuffer::from_iter([false, false, true, true, false]);
+        let validity = validity_set(Some(&valid_island), valid_island.len(), "test").unwrap();
+        assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions);
+        assert!(matches!(
+            validity.positions,
+            SparsePositionSet::Range { start: 2, len: 2 }
+        ));
+
+        let all_valid = BooleanBuffer::from_iter([true, true, true]);
+        let validity = validity_set(Some(&all_valid), all_valid.len(), "test").unwrap();
+        assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions);
+        assert!(matches!(validity.positions, SparsePositionSet::Empty));
+
+        let all_null = BooleanBuffer::from_iter([false, false, false]);
+        let validity = validity_set(Some(&all_null), all_null.len(), "test").unwrap();
+        assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions);
+        assert!(matches!(validity.positions, SparsePositionSet::Empty));
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_nullable_primitive_roundtrip() {
+        let array = Arc::new(Int32Array::from(vec![
+            Some(10),
+            None,
+            Some(20),
+            Some(30),
+            None,
+            Some(40),
+        ])) as ArrayRef;
+        let pages = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert_eq!(pages.len(), 1);
+        let sparse = sparse_layout(&pages[0]);
+        assert_eq!(sparse.num_items, 6);
+        assert_eq!(sparse.num_visible_items, 6);
+        assert_eq!(sparse.structural_layers.len(), 1);
+        let validity = validity_layer(&sparse.structural_layers[0])
+            .unwrap()
+            .validity
+            .as_ref()
+            .unwrap();
+        assert_eq!(
+            validity.meaning,
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32
+        );
+        assert!(matches!(
+            validity.positions.as_ref().unwrap().positions,
+            Some(pb21::sparse_position_set::Positions::Explicit(_))
+        ));
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..5)
+            .with_indices(vec![0, 2, 5]);
+        check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_nullable_struct_roundtrip() {
+        let array = nullable_struct();
+        let pages = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert_eq!(pages.len(), 1);
+        let sparse = sparse_layout(&pages[0]);
+        assert_eq!(sparse.structural_layers.len(), 2);
+        assert!(
+            sparse
+                .structural_layers
+                .iter()
+                .all(|layer| validity_layer(layer).is_some())
+        );
+        let struct_validity = validity_layer(&sparse.structural_layers[0])
+            .unwrap()
+            .validity
+            .as_ref()
+            .unwrap();
+        assert_eq!(
+            struct_validity.meaning,
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32
+        );
+        assert!(matches!(
+            struct_validity.positions.as_ref().unwrap().positions,
+            Some(pb21::sparse_position_set::Positions::Explicit(_))
+        ));
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..5)
+            .with_indices(vec![0, 2, 4]);
+        check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_struct_with_constant_and_sparse_children() {
+        let fields = Fields::from(vec![
+            ArrowField::new("constant", DataType::Int32, true),
+            ArrowField::new("sparse", DataType::Int32, true),
+        ]);
+        let array = Arc::new(StructArray::new(
+            fields,
+            vec![
+                Arc::new(Int32Array::from(vec![None::<i32>; 5])),
+                Arc::new(Int32Array::from(vec![
+                    Some(10),
+                    Some(20),
+                    None,
+                    Some(40),
+                    Some(50),
+                ])),
+            ],
+            Some(null_buffer([true, false, true, true, false])),
+        )) as ArrayRef;
+        let pages = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(pages.iter().any(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::ConstantLayout(_)
+        )));
+        assert!(pages.iter().any(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::SparseLayout(_)
+        )));
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..5)
+            .with_indices(vec![0, 2, 4]);
+        check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_emits_both_validity_polarities() {
+        let mostly_valid = Arc::new(Int32Array::from(vec![
+            Some(0),
+            None,
+            Some(2),
+            Some(3),
+            None,
+            Some(5),
+        ])) as ArrayRef;
+        let mostly_null = Arc::new(Int32Array::from(vec![
+            None,
+            Some(1),
+            None,
+            None,
+            Some(4),
+            None,
+        ])) as ArrayRef;
+
+        let null_positions = encode_pages(
+            mostly_valid.clone(),
+            LanceFileVersion::V2_3,
+            sparse_metadata(),
+        )
+        .await
+        .unwrap();
+        let validity = validity_layer(&sparse_layout(&null_positions[0]).structural_layers[0])
+            .unwrap()
+            .validity
+            .as_ref()
+            .unwrap();
+        assert_eq!(
+            validity.meaning,
+            pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32
+        );
+
+        let valid_positions = encode_pages(
+            mostly_null.clone(),
+            LanceFileVersion::V2_3,
+            sparse_metadata(),
+        )
+        .await
+        .unwrap();
+        let validity = validity_layer(&sparse_layout(&valid_positions[0]).structural_layers[0])
+            .unwrap()
+            .validity
+            .as_ref()
+            .unwrap();
+        assert_eq!(
+            validity.meaning,
+            pb21::sparse_validity_set::Meaning::SparseValidityValidPositions as i32
+        );
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..5)
+            .with_indices(vec![0, 2, 5]);
+        for array in [mostly_valid, mostly_null] {
+            check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_nested_page_boundaries_range_and_take() {
+        let nested = deeply_nested();
+        let chunks = vec![nested.slice(0, 2), nested.slice(2, 3)];
+        let pages = encode_chunks(chunks.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(pages.len() >= 2, "expected multiple sparse pages");
+        let sparse = pages
+            .iter()
+            .map(sparse_layout)
+            .collect::<Vec<&pb21::SparseLayout>>();
+        assert!(sparse.iter().any(|layout| {
+            layout
+                .structural_layers
+                .iter()
+                .filter(|layer| {
+                    matches!(
+                        layer.layer.as_ref(),
+                        Some(pb21::sparse_structural_layer::Layer::List(_))
+                    )
+                })
+                .count()
+                >= 2
+        }));
+        assert!(sparse.iter().any(|layout| {
+            layout
+                .structural_layers
+                .iter()
+                .any(|layer| validity_layer(layer).is_some())
+        }));
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_batch_size(2)
+            .with_range(1..5)
+            .with_range(2..4)
+            .with_indices(vec![0, 2, 4])
+            .with_indices(vec![1, 3]);
+        check_round_trip_encoding_of_data(chunks, &cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_list_and_large_list_null_empty_roundtrip() {
+        let list = list_i32(
+            vec![0, 0, 0, 2, 3, 3],
+            Some(vec![false, true, true, true, true]),
+        );
+        let large_list = large_list_i32(
+            vec![0, 0, 0, 2, 3, 3],
+            Some(vec![false, true, true, true, true]),
+        );
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(0..5)
+            .with_range(1..4)
+            .with_indices(vec![0, 1, 4])
+            .with_indices(vec![2, 3]);
+
+        for array in [list, large_list] {
+            let pages = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+                .await
+                .unwrap();
+            assert!(pages.iter().map(sparse_layout).all(|layout| {
+                layout.structural_layers.iter().any(|layer| {
+                    matches!(
+                        layer.layer.as_ref(),
+                        Some(pb21::sparse_structural_layer::Layer::List(_))
+                    )
+                })
+            }));
+            check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_map_and_fixed_size_list_roundtrip() {
+        let map = map_i32();
+        let map_pages = encode_pages(map.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(map_pages.iter().map(sparse_layout).any(|layout| {
+            layout.structural_layers.iter().any(|layer| {
+                matches!(
+                    layer.layer.as_ref(),
+                    Some(pb21::sparse_structural_layer::Layer::List(_))
+                )
+            })
+        }));
+        let map_cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..4)
+            .with_indices(vec![0, 2, 3]);
+        check_round_trip_encoding_of_data(vec![map], &map_cases, sparse_metadata()).await;
+
+        let fsl = fixed_size_list_struct();
+        let fsl_pages = encode_pages(fsl.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        for page in &fsl_pages {
+            let layout = sparse_layout(page);
+            let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap());
+            let fixed_size_scale = layout
+                .structural_layers
+                .iter()
+                .filter_map(fixed_size_list_dimension)
+                .product::<u64>();
+            assert_eq!(page.num_rows, outer_slots * fixed_size_scale);
+        }
+        assert!(fsl_pages.iter().map(sparse_layout).any(|layout| {
+            layout
+                .structural_layers
+                .iter()
+                .any(|layer| fixed_size_list_dimension(layer) == Some(2))
+        }));
+        let fsl_cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..6)
+            .with_indices(vec![0, 3, 5]);
+        check_round_trip_encoding_of_data(vec![fsl], &fsl_cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_list_fixed_size_list_struct_roundtrip() {
+        let array = list_fixed_size_list_struct();
+        let pages = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(pages.iter().map(sparse_layout).any(|layout| {
+            let kinds = layout
+                .structural_layers
+                .iter()
+                .map(|layer| match layer.layer.as_ref().unwrap() {
+                    pb21::sparse_structural_layer::Layer::Validity(_) => "validity",
+                    pb21::sparse_structural_layer::Layer::List(_) => "list",
+                    pb21::sparse_structural_layer::Layer::FixedSizeList(_) => "fixed-size-list",
+                })
+                .collect::<Vec<_>>();
+            kinds.starts_with(&["list", "fixed-size-list"])
+        }));
+        for page in &pages {
+            let layout = sparse_layout(page);
+            let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap());
+            assert_eq!(page.num_rows, outer_slots * 2);
+        }
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..4)
+            .with_indices(vec![0, 2, 3])
+            .with_indices(vec![1, 3]);
+        check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+    }
+
+    #[tokio::test]
+    async fn test_explicit_sparse_serializes_semantic_list_forms() {
+        let all_array = list_i32(vec![0, 2, 4, 6], None);
+        let all = encode_pages(all_array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        let all_layer = list_layer(sparse_layout(&all[0]));
+        assert!(matches!(
+            all_layer.non_empty_positions.as_ref().unwrap().positions,
+            Some(pb21::sparse_position_set::Positions::All(_))
+        ));
+        assert!(matches!(
+            all_layer.counts.as_ref().unwrap().counts,
+            Some(pb21::sparse_count_set::Counts::Constant(
+                pb21::SparseCountConstant { value: 2 }
+            ))
+        ));
+        assert_eq!(all[0].data.len(), 2);
+
+        let range_array = list_i32(vec![0, 2, 4, 4, 4], None);
+        let range = encode_pages(
+            range_array.clone(),
+            LanceFileVersion::V2_3,
+            sparse_metadata(),
+        )
+        .await
+        .unwrap();
+        let range_layer = list_layer(sparse_layout(&range[0]));
+        assert!(matches!(
+            range_layer.non_empty_positions.as_ref().unwrap().positions,
+            Some(pb21::sparse_position_set::Positions::Range(
+                pb21::SparsePositionRange {
+                    start: 0,
+                    length: 2
+                }
+            ))
+        ));
+        assert!(matches!(
+            range_layer.counts.as_ref().unwrap().counts,
+            Some(pb21::sparse_count_set::Counts::Constant(
+                pb21::SparseCountConstant { value: 2 }
+            ))
+        ));
+        assert_eq!(range[0].data.len(), 2);
+
+        let explicit_array = list_i32(vec![0, 1, 1, 4, 4, 6], None);
+        let explicit = encode_pages(
+            explicit_array.clone(),
+            LanceFileVersion::V2_3,
+            sparse_metadata(),
+        )
+        .await
+        .unwrap();
+        let explicit_layer = list_layer(sparse_layout(&explicit[0]));
+        assert!(matches!(
+            explicit_layer
+                .non_empty_positions
+                .as_ref()
+                .unwrap()
+                .positions,
+            Some(pb21::sparse_position_set::Positions::Explicit(_))
+        ));
+        assert!(matches!(
+            explicit_layer.counts.as_ref().unwrap().counts,
+            Some(pb21::sparse_count_set::Counts::Explicit(_))
+        ));
+        assert_eq!(explicit[0].data.len(), 4);
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1])
+            .with_range(1..3)
+            .with_indices(vec![0, 2]);
+        for array in [all_array, range_array, explicit_array] {
+            check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_constant_layout_boundary_is_explicit() {
+        let structural_only = encode_pages(
+            list_i32(vec![0, 0, 0, 0], None),
+            LanceFileVersion::V2_3,
+            sparse_metadata(),
+        )
+        .await
+        .unwrap();
+        assert!(matches!(
+            page_layout(&structural_only[0]),
+            pb21::page_layout::Layout::ConstantLayout(_)
+        ));
+
+        let empty_struct = Arc::new(StructArray::new_empty_fields(3, None)) as ArrayRef;
+        let empty_struct_pages =
+            encode_pages(empty_struct, LanceFileVersion::V2_3, sparse_metadata())
+                .await
+                .unwrap();
+        assert!(matches!(
+            page_layout(&empty_struct_pages[0]),
+            pb21::page_layout::Layout::ConstantLayout(_)
+        ));
+
+        let all_null = Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef;
+        let all_null_pages = encode_pages(all_null, LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(matches!(
+            page_layout(&all_null_pages[0]),
+            pb21::page_layout::Layout::ConstantLayout(_)
+        ));
+
+        let constant = Arc::new(Int32Array::from(vec![7, 7, 7])) as ArrayRef;
+        let constant_pages = encode_pages(constant, LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(matches!(
+            page_layout(&constant_pages[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_default_and_explicit_dense_layouts_are_unchanged() {
+        let array = Arc::new(Int32Array::from_iter_values(0..16)) as ArrayRef;
+        let v2_2_default = encode_pages(array.clone(), LanceFileVersion::V2_2, HashMap::new())
+            .await
+            .unwrap();
+        let v2_2_miniblock = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_2,
+            structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK),
+        )
+        .await
+        .unwrap();
+        assert_eq!(v2_2_default.len(), v2_2_miniblock.len());
+        for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) {
+            assert!(matches!(
+                page_layout(default),
+                pb21::page_layout::Layout::MiniBlockLayout(_)
+            ));
+            assert_eq!(page_layout(default), page_layout(explicit));
+            assert_eq!(default.data, explicit.data);
+        }
+
+        let v2_3_default = encode_pages(array.clone(), LanceFileVersion::V2_3, HashMap::new())
+            .await
+            .unwrap();
+        assert!(matches!(
+            page_layout(&v2_3_default[0]),
+            pb21::page_layout::Layout::MiniBlockLayout(_)
+        ));
+
+        let v2_3_miniblock = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK),
+        )
+        .await
+        .unwrap();
+        assert!(matches!(
+            page_layout(&v2_3_miniblock[0]),
+            pb21::page_layout::Layout::MiniBlockLayout(_)
+        ));
+
+        let v2_3_fullzip = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_FULLZIP),
+        )
+        .await
+        .unwrap();
+        assert!(matches!(
+            page_layout(&v2_3_fullzip[0]),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        ));
+
+        let v2_3_sparse = encode_pages(array, LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert!(matches!(
+            page_layout(&v2_3_sparse[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_auto_sparse_for_split_required_page() {
+        const NUM_ROWS: usize = 70_000;
+        let array = sparse_i32_list(NUM_ROWS, 2_000);
+
+        let within_budget = encode_pages(
+            sparse_i32_list(4_096, 1_024),
+            LanceFileVersion::V2_3,
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(within_budget.len(), 1);
+        assert!(matches!(
+            page_layout(&within_budget[0]),
+            pb21::page_layout::Layout::MiniBlockLayout(_)
+        ));
+
+        let automatic = encode_pages(array.clone(), LanceFileVersion::V2_3, HashMap::new())
+            .await
+            .unwrap();
+        assert_eq!(automatic.len(), 1);
+        assert!(matches!(
+            page_layout(&automatic[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+
+        let miniblock = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK),
+        )
+        .await
+        .unwrap();
+        assert!(miniblock.len() > 1);
+        assert!(miniblock.iter().all(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::MiniBlockLayout(_)
+        )));
+
+        let fullzip = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_FULLZIP),
+        )
+        .await
+        .unwrap();
+        assert_eq!(fullzip.len(), miniblock.len());
+        assert!(fullzip.iter().all(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        )));
+
+        let sparse = encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+            .await
+            .unwrap();
+        assert_eq!(sparse.len(), 1);
+        assert!(matches!(
+            page_layout(&sparse[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+
+        let v2_2_default = encode_pages(array.clone(), LanceFileVersion::V2_2, HashMap::new())
+            .await
+            .unwrap();
+        let v2_2_miniblock = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_2,
+            structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK),
+        )
+        .await
+        .unwrap();
+        assert_eq!(v2_2_default.len(), v2_2_miniblock.len());
+        for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) {
+            assert_eq!(page_layout(default), page_layout(explicit));
+            assert_eq!(default.data, explicit.data);
+        }
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1024 * 1024])
+            .with_batch_size(NUM_ROWS as u32)
+            .with_range(1_999..2_002)
+            .with_indices(vec![0, 1_999, 2_000, NUM_ROWS as u64 - 1]);
+        check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await;
+    }
+
+    #[tokio::test]
+    async fn test_auto_sparse_for_single_row_over_budget() {
+        let array = unsplittable_nested_list(
+            Arc::new(Int32Array::from(vec![42, 43])),
+            Arc::new(ArrowField::new("item", DataType::Int32, true)),
+        );
+
+        let automatic = encode_pages(array.clone(), LanceFileVersion::V2_3, HashMap::new())
+            .await
+            .unwrap();
+        assert_eq!(automatic.len(), 1);
+        assert!(matches!(
+            page_layout(&automatic[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+
+        let v2_2 = encode_pages(array.clone(), LanceFileVersion::V2_2, HashMap::new())
+            .await
+            .unwrap();
+        assert_eq!(v2_2.len(), 1);
+        assert!(matches!(
+            page_layout(&v2_2[0]),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        ));
+
+        let miniblock_error = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK),
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            miniblock_error
+                .to_string()
+                .contains("Mini-block cannot encode 70000 rep/def levels")
+        );
+
+        let fullzip = encode_pages(
+            array.clone(),
+            LanceFileVersion::V2_3,
+            structural_metadata(STRUCTURAL_ENCODING_FULLZIP),
+        )
+        .await
+        .unwrap();
+        assert!(matches!(
+            page_layout(&fullzip[0]),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        ));
+
+        let explicit_sparse =
+            encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+                .await
+                .unwrap();
+        assert!(matches!(
+            page_layout(&explicit_sparse[0]),
+            pb21::page_layout::Layout::SparseLayout(_)
+        ));
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_range(0..1)
+            .with_indices(vec![0]);
+        check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await;
+    }
+
+    #[tokio::test]
+    async fn test_auto_sparse_keeps_unsupported_values_dense() {
+        const NUM_ROWS: usize = 70_000;
+        let num_values = NUM_ROWS.div_ceil(2_000);
+
+        let (dictionary, dictionary_field) = dictionary_values(num_values);
+        let dictionary_array = sparse_list_values(NUM_ROWS, 2_000, dictionary, dictionary_field);
+        let dictionary_pages = encode_pages(
+            dictionary_array.clone(),
+            LanceFileVersion::V2_3,
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+        assert!(dictionary_pages.len() > 1);
+        assert!(dictionary_pages.iter().all(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::MiniBlockLayout(_)
+        )));
+
+        let (packed_values, packed_field) = variable_packed_struct_values(num_values);
+        let packed_array = sparse_list_values(NUM_ROWS, 2_000, packed_values, packed_field);
+        let packed_pages =
+            encode_pages(packed_array.clone(), LanceFileVersion::V2_3, HashMap::new())
+                .await
+                .unwrap();
+        assert!(packed_pages.len() > 1);
+        assert!(packed_pages.iter().all(|page| matches!(
+            page_layout(page),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        )));
+
+        let (packed_values, packed_field) = variable_packed_struct_values(2);
+        let unsplittable_packed = unsplittable_nested_list(packed_values, packed_field);
+        let packed_fallback = encode_pages(
+            unsplittable_packed.clone(),
+            LanceFileVersion::V2_3,
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(packed_fallback.len(), 1);
+        assert!(matches!(
+            page_layout(&packed_fallback[0]),
+            pb21::page_layout::Layout::FullZipLayout(_)
+        ));
+
+        let (dictionary, dictionary_field) = dictionary_values(2);
+        let unsplittable_dictionary = unsplittable_nested_list(dictionary, dictionary_field);
+        let v2_2_error = encode_pages(
+            unsplittable_dictionary.clone(),
+            LanceFileVersion::V2_2,
+            HashMap::new(),
+        )
+        .await
+        .unwrap_err();
+        let v2_3_error = encode_pages(
+            unsplittable_dictionary,
+            LanceFileVersion::V2_3,
+            HashMap::new(),
+        )
+        .await
+        .unwrap_err();
+        assert_eq!(v2_3_error.to_string(), v2_2_error.to_string());
+        assert!(
+            v2_3_error
+                .to_string()
+                .contains("Mini-block cannot encode 70000 rep/def levels")
+        );
+
+        let cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1024 * 1024])
+            .with_batch_size(NUM_ROWS as u32)
+            .with_range(1_999..2_002)
+            .with_indices(vec![0, 2_000, NUM_ROWS as u64 - 1]);
+        check_round_trip_encoding_of_data(vec![dictionary_array], &cases, HashMap::new()).await;
+        let packed_cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1024 * 1024])
+            .with_batch_size(NUM_ROWS as u32);
+        check_round_trip_encoding_of_data(vec![packed_array], &packed_cases, HashMap::new()).await;
+
+        let single_row_cases = TestCases::default()
+            .with_min_file_version(LanceFileVersion::V2_3)
+            .with_max_file_version(LanceFileVersion::V2_3)
+            .with_page_sizes(vec![1024 * 1024])
+            .with_range(0..1)
+            .with_indices(vec![0]);
+        check_round_trip_encoding_of_data(
+            vec![unsplittable_packed],
+            &single_row_cases,
+            HashMap::new(),
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn test_auto_sparse_wide_values_keep_dense_fallback() {
+        let first = vec![0xAB_u8; 5_000];
+        let second = vec![0xCD_u8; 5_000];
+        let fixed_size_binary = Arc::new(
+            FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                [Some(first.as_slice()), Some(second.as_slice())].into_iter(),
+                5_000,
+            )
+            .unwrap(),
+        ) as ArrayRef;
+
+        const FSL_DIMENSION: i32 = 2_048;
+        let fixed_size_list = Arc::new(
+            FixedSizeListArray::try_new(
+                Arc::new(ArrowField::new("item", DataType::Int32, true)),
+                FSL_DIMENSION,
+                Arc::new(Int32Array::from_iter_values(0..(FSL_DIMENSION * 2))),
+                None,
+            )
+            .unwrap(),
+        ) as ArrayRef;
+
+        for (label, values) in [
+            ("fixed-size binary", fixed_size_binary),
+            ("fixed-size list", fixed_size_list),
+        ] {
+            let item_field = Arc::new(ArrowField::new("item", values.data_type().clone(), true));
+            let array = unsplittable_nested_list(values, item_field);
+
+            let v2_2 = encode_pages(array.clone(), LanceFileVersion::V2_2, HashMap::new())
+                .await
+                .unwrap();
+            let v2_3 = encode_pages(array.clone(), LanceFileVersion::V2_3, HashMap::new())
+                .await
+                .unwrap();
+            for pages in [&v2_2, &v2_3] {
+                assert_eq!(pages.len(), 1, "unexpected {label} page count");
+                assert!(
+                    matches!(
+                        page_layout(&pages[0]),
+                        pb21::page_layout::Layout::FullZipLayout(_)
+                    ),
+                    "{label} should retain the dense full-zip fallback"
+                );
+            }
+
+            let explicit_error =
+                encode_pages(array.clone(), LanceFileVersion::V2_3, sparse_metadata())
+                    .await
+                    .unwrap_err();
+            assert!(
+                explicit_error.to_string().contains("too wide"),
+                "explicit sparse should preserve the {label} value error: {explicit_error}"
+            );
+
+            let cases = TestCases::default()
+                .with_min_file_version(LanceFileVersion::V2_2)
+                .with_max_file_version(LanceFileVersion::V2_3)
+                .with_range(0..1)
+                .with_indices(vec![0]);
+            check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await;
+        }
+    }
+
+    #[test]
+    fn test_explicit_sparse_rejects_lance_2_2() {
+        let array = Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef;
+        let Err(error) = create_encoder(&array, LanceFileVersion::V2_2, sparse_metadata()) else {
+            panic!("expected Lance 2.2 to reject explicit sparse encoding");
+        };
+        assert!(
+            error
+                .to_string()
+                .contains("requires Lance file format 2.3+")
+        );
+
+        let structural_only = list_i32(vec![0, 0, 0], None);
+        let Err(error) =
+            create_encoder(&structural_only, LanceFileVersion::V2_2, sparse_metadata())
+        else {
+            panic!("expected Lance 2.2 structural-only input to reject explicit sparse encoding");
+        };
+        assert!(
+            error
+                .to_string()
+                .contains("requires Lance file format 2.3+")
+        );
+    }
+}
diff --git a/rust/lance-encoding/src/encodings/logical/struct.rs b/rust/lance-encoding/src/encodings/logical/struct.rs
index e8c6289275a..dc734da72b5 100644
--- a/rust/lance-encoding/src/encodings/logical/struct.rs
+++ b/rust/lance-encoding/src/encodings/logical/struct.rs
@@ -368,26 +368,57 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask {
             .map(|task| task.decode())
             .collect::<Result<Vec<_>>>()?;
         let mut children = Vec::with_capacity(arrays.len());
+        let mut repdefs = Vec::with_capacity(arrays.len());
         let mut data_size = 0u64;
         let mut arrays_iter = arrays.into_iter();
-        let first_array = arrays_iter.next().unwrap();
+        let first_array = arrays_iter.next().ok_or_else(|| {
+            Error::internal("Struct decoder unexpectedly has no child arrays".to_string())
+        })?;
         let length = first_array.array.len();
 
         // The repdef should be identical across all children at this point
-        let mut repdef = first_array.repdef;
+        repdefs.push(first_array.repdef);
         data_size += first_array.data_size;
         children.push(first_array.array);
 
         for array in arrays_iter {
-            debug_assert_eq!(length, array.array.len());
+            if length != array.array.len() {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Struct child array length {} does not match sibling length {}",
+                        array.array.len(),
+                        length
+                    )
+                    .into(),
+                ));
+            }
             data_size += array.data_size;
             children.push(array.array);
+            repdefs.push(array.repdef);
+        }
+
+        // Dense rep/def state can retain child-specific repetition information after a child
+        // decoder finishes, so comparing dense siblings is not meaningful. If any child carries
+        // sparse state, keep a sparse child as the canonical structural plan and compare it with
+        // every other sparse sibling so sparse metadata is never silently discarded.
+        let primary_repdef = repdefs
+            .iter()
+            .position(CompositeRepDefUnraveler::has_sparse)
+            .unwrap_or(0);
+        let mut repdef = repdefs.swap_remove(primary_repdef);
+        if repdef.has_sparse() {
+            for sibling in repdefs {
+                if sibling.has_sparse() {
+                    repdef.add_compatibility_check(sibling);
+                }
+            }
         }
 
         let validity = if self.is_root {
+            repdef.ensure_exhausted()?;
             None
         } else {
-            repdef.unravel_validity(length)
+            repdef.unravel_validity(length)?
         };
 
         let array = StructArray::try_new(self.child_fields, children, validity)
diff --git a/rust/lance-encoding/src/encodings/physical/bitpacking.rs b/rust/lance-encoding/src/encodings/physical/bitpacking.rs
index 8ebdcc13c56..be0b747e7dc 100644
--- a/rust/lance-encoding/src/encodings/physical/bitpacking.rs
+++ b/rust/lance-encoding/src/encodings/physical/bitpacking.rs
@@ -241,6 +241,15 @@ impl MiniBlockDecompressor for InlineBitpacking {
     fn decompress(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
         assert_eq!(data.len(), 1);
         let data = data.into_iter().next().unwrap();
+        if num_values == 0 {
+            // Empty mini-blocks have no inline bit-width header to decode.
+            return Ok(DataBlock::FixedWidth(FixedWidthDataBlock {
+                data: LanceBuffer::empty(),
+                bits_per_value: self.uncompressed_bit_width,
+                num_values: 0,
+                block_info: BlockInfo::new(),
+            }));
+        }
         match self.uncompressed_bit_width {
             8 => Self::unchunk::<u8>(data, num_values),
             16 => Self::unchunk::<u16>(data, num_values),
@@ -528,15 +537,36 @@ mod test {
 
     use arrow_array::{Array, Int8Array, Int64Array};
     use arrow_schema::DataType;
+    use rstest::rstest;
 
-    use super::{ELEMS_PER_CHUNK, bitpack_out_of_line, unpack_out_of_line};
+    use super::{ELEMS_PER_CHUNK, InlineBitpacking, bitpack_out_of_line, unpack_out_of_line};
     use crate::{
         buffer::LanceBuffer,
-        data::{BlockInfo, FixedWidthDataBlock},
+        compression::MiniBlockDecompressor,
+        data::{BlockInfo, DataBlock, FixedWidthDataBlock},
         testing::{TestCases, check_round_trip_encoding_of_data},
         version::LanceFileVersion,
     };
 
+    #[rstest]
+    #[case::u8(8)]
+    #[case::u16(16)]
+    #[case::u32(32)]
+    #[case::u64(64)]
+    fn test_inline_bitpacking_decompress_empty_miniblock(#[case] bit_width: u64) {
+        let decompressor = InlineBitpacking::new(bit_width);
+        let decompressed =
+            MiniBlockDecompressor::decompress(&decompressor, vec![LanceBuffer::empty()], 0)
+                .unwrap();
+
+        let DataBlock::FixedWidth(block) = decompressed else {
+            panic!("Expected FixedWidth block");
+        };
+        assert_eq!(block.bits_per_value, bit_width);
+        assert_eq!(block.num_values, 0);
+        assert_eq!(block.data.len(), 0);
+    }
+
     #[test_log::test(tokio::test)]
     async fn test_miniblock_bitpack() {
         let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1);
diff --git a/rust/lance-encoding/src/encodings/physical/packed.rs b/rust/lance-encoding/src/encodings/physical/packed.rs
index 3ade6a70818..ad2221dffed 100644
--- a/rust/lance-encoding/src/encodings/physical/packed.rs
+++ b/rust/lance-encoding/src/encodings/physical/packed.rs
@@ -323,7 +323,7 @@ impl PerValueCompressor for PackedStructVariablePerValueEncoder {
         let mut field_data = Vec::with_capacity(self.fields.len());
         let mut field_metadata = Vec::with_capacity(self.fields.len());
 
-        for (field, child_block) in self.fields.iter().zip(struct_block.children.into_iter()) {
+        for (field, child_block) in self.fields.iter().zip(struct_block.children) {
             let compressor = crate::compression::CompressionStrategy::create_per_value(
                 &self.strategy,
                 field,
@@ -688,7 +688,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor {
         }
 
         let mut children = Vec::with_capacity(self.fields.len());
-        for (field, accumulator) in self.fields.iter().zip(accumulators.into_iter()) {
+        for (field, accumulator) in self.fields.iter().zip(accumulators) {
             match (field, accumulator) {
                 (
                     VariablePackedStructFieldDecoder {
diff --git a/rust/lance-encoding/src/encodings/physical/rle.rs b/rust/lance-encoding/src/encodings/physical/rle.rs
index da758b05bfe..145b9c43779 100644
--- a/rust/lance-encoding/src/encodings/physical/rle.rs
+++ b/rust/lance-encoding/src/encodings/physical/rle.rs
@@ -65,6 +65,7 @@ use crate::encodings::logical::primitive::miniblock::{
     MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed,
     MiniBlockCompressor,
 };
+use crate::encodings::physical::block::{CompressionConfig, GeneralBufferCompressor};
 use crate::format::ProtobufUtils21;
 use crate::format::pb21::CompressiveEncoding;
 
@@ -326,6 +327,18 @@ pub(crate) fn accumulate_run_length_entries(
 #[derive(Debug)]
 pub struct RleEncoder {
     run_length_width: RunLengthWidth,
+    values_compression: Option<CompressionConfig>,
+    run_lengths_compression: Option<CompressionConfig>,
+    use_child_bitpacking: bool,
+}
+
+#[derive(Clone)]
+struct RleChildCandidate {
+    encoding: CompressiveEncoding,
+    data: LanceBuffer,
+    chunk_sizes: Vec<u32>,
+    size: usize,
+    requires_num_values: bool,
 }
 
 impl Default for RleEncoder {
@@ -338,11 +351,33 @@ impl RleEncoder {
     pub fn new() -> Self {
         Self {
             run_length_width: RunLengthWidth::U8,
+            values_compression: None,
+            run_lengths_compression: None,
+            use_child_bitpacking: false,
         }
     }
 
     pub(crate) fn with_run_length_width(run_length_width: RunLengthWidth) -> Self {
-        Self { run_length_width }
+        Self {
+            run_length_width,
+            values_compression: None,
+            run_lengths_compression: None,
+            use_child_bitpacking: false,
+        }
+    }
+
+    pub(crate) fn with_child_encoding(
+        run_length_width: RunLengthWidth,
+        values_compression: Option<CompressionConfig>,
+        run_lengths_compression: Option<CompressionConfig>,
+        use_child_bitpacking: bool,
+    ) -> Self {
+        Self {
+            run_length_width,
+            values_compression,
+            run_lengths_compression,
+            use_child_bitpacking,
+        }
     }
 
     fn encode_data(
@@ -417,7 +452,15 @@ impl RleEncoder {
             };
 
             if values_processed == 0 {
-                break;
+                // A non-final chunk needs at least two values because log_num_values == 0
+                // identifies the final chunk. Report an error instead of returning partial data.
+                return Err(Error::internal(format!(
+                    "RLE encoder made no progress: values_remaining={values_remaining}, \
+                     offset={offset}, data_len={}, bits_per_value={bits_per_value}, \
+                     max_miniblock_values={}",
+                    data.len(),
+                    *MAX_MINIBLOCK_VALUES
+                )));
             }
 
             let log_num_values = if is_last_chunk {
@@ -678,6 +721,318 @@ impl RleEncoder {
 
         total_chunks * (type_size + self.run_length_width.bytes_per_value())
     }
+
+    fn flat_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+    ) -> RleChildCandidate {
+        RleChildCandidate {
+            encoding: ProtobufUtils21::flat(bits_per_value, None),
+            data: buffers[buffer_index].clone(),
+            chunk_sizes: chunks
+                .iter()
+                .map(|chunk| chunk.buffer_sizes[buffer_index])
+                .collect(),
+            size: buffers[buffer_index].len(),
+            requires_num_values: false,
+        }
+    }
+
+    fn general_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+        compression: CompressionConfig,
+    ) -> Result<Option<RleChildCandidate>> {
+        if buffers.is_empty() || buffers[buffer_index].is_empty() {
+            return Ok(None);
+        };
+
+        let compressor = GeneralBufferCompressor::get_compressor(compression)?;
+        let original = &buffers[buffer_index];
+        let mut compressed = Vec::new();
+        let mut offset = 0usize;
+        let mut total_original_size = 0usize;
+        let mut compressed_sizes = Vec::with_capacity(chunks.len());
+
+        for chunk in chunks.iter() {
+            let chunk_size = chunk.buffer_sizes[buffer_index] as usize;
+            let end = offset.checked_add(chunk_size).ok_or_else(|| {
+                Error::invalid_input_source("RLE child buffer offset overflow".into())
+            })?;
+            if end > original.len() {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}",
+                        buffer_index,
+                        end,
+                        original.len()
+                    )
+                    .into(),
+                ));
+            }
+
+            let start = compressed.len();
+            compressor.compress(&original.as_ref()[offset..end], &mut compressed)?;
+            let compressed_size = compressed.len() - start;
+            let compressed_size = u32::try_from(compressed_size).map_err(|_| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} compressed chunk is too large: {} bytes",
+                        buffer_index, compressed_size
+                    )
+                    .into(),
+                )
+            })?;
+            compressed_sizes.push(compressed_size);
+            total_original_size += chunk_size;
+            offset = end;
+        }
+
+        if compressed.len() >= total_original_size {
+            return Ok(None);
+        }
+
+        let encoding =
+            ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(bits_per_value, None))?;
+        Ok(Some(
+            RleChildCandidate {
+                encoding,
+                data: LanceBuffer::from(compressed),
+                chunk_sizes: compressed_sizes,
+                size: 0,
+                requires_num_values: false,
+            }
+            .with_size_from_data(),
+        ))
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn bitpacked_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+    ) -> Result<Option<RleChildCandidate>> {
+        let original = &buffers[buffer_index];
+        if original.is_empty() {
+            return Ok(None);
+        }
+        let packed_bits = Self::required_bits(original, bits_per_value)?;
+        if packed_bits >= bits_per_value {
+            return Ok(None);
+        }
+
+        let compressor = crate::encodings::physical::bitpacking::OutOfLineBitpacking::new(
+            packed_bits,
+            bits_per_value,
+        );
+        let mut packed = Vec::new();
+        let mut offset = 0usize;
+        let mut packed_sizes = Vec::with_capacity(chunks.len());
+        let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE child bit width is too large: {bits_per_value}").into(),
+            )
+        })?;
+
+        for chunk in chunks {
+            let chunk_size = chunk.buffer_sizes[buffer_index] as usize;
+            let end = offset.checked_add(chunk_size).ok_or_else(|| {
+                Error::invalid_input_source("RLE child buffer offset overflow".into())
+            })?;
+            if end > original.len() {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}",
+                        buffer_index,
+                        end,
+                        original.len()
+                    )
+                    .into(),
+                ));
+            }
+            if bytes_per_value == 0 || !chunk_size.is_multiple_of(bytes_per_value) {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk has invalid size {} for {} bits per value",
+                        buffer_index, chunk_size, bits_per_value
+                    )
+                    .into(),
+                ));
+            }
+
+            let child_values = (chunk_size / bytes_per_value) as u64;
+            let block = DataBlock::FixedWidth(FixedWidthDataBlock {
+                bits_per_value,
+                data: original.slice_with_length(offset, chunk_size),
+                num_values: child_values,
+                block_info: BlockInfo::default(),
+            });
+            let chunk_packed = BlockCompressor::compress(&compressor, block)?;
+            let packed_size = u32::try_from(chunk_packed.len()).map_err(|_| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} bitpacked chunk is too large: {} bytes",
+                        buffer_index,
+                        chunk_packed.len()
+                    )
+                    .into(),
+                )
+            })?;
+            packed_sizes.push(packed_size);
+            packed.extend_from_slice(chunk_packed.as_ref());
+            offset = end;
+        }
+
+        if packed.len() >= original.len() {
+            return Ok(None);
+        }
+
+        Ok(Some(
+            RleChildCandidate {
+                encoding: ProtobufUtils21::out_of_line_bitpacking(
+                    bits_per_value,
+                    ProtobufUtils21::flat(packed_bits, None),
+                ),
+                data: LanceBuffer::from(packed),
+                chunk_sizes: packed_sizes,
+                size: 0,
+                requires_num_values: true,
+            }
+            .with_size_from_data(),
+        ))
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn required_bits(buffer: &LanceBuffer, bits_per_value: u64) -> Result<u64> {
+        let max_value = match bits_per_value {
+            8 => buffer.as_ref().iter().map(|value| *value as u64).max(),
+            16 => buffer
+                .as_ref()
+                .chunks_exact(2)
+                .map(|value| u16::from_le_bytes(value.try_into().unwrap()) as u64)
+                .max(),
+            32 => buffer
+                .as_ref()
+                .chunks_exact(4)
+                .map(|value| u32::from_le_bytes(value.try_into().unwrap()) as u64)
+                .max(),
+            64 => buffer
+                .as_ref()
+                .chunks_exact(8)
+                .map(|value| u64::from_le_bytes(value.try_into().unwrap()))
+                .max(),
+            _ => {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child bitpacking only supports 8, 16, 32, or 64-bit values, got {bits_per_value}"
+                    )
+                    .into(),
+                ));
+            }
+        }
+        .unwrap_or(0);
+        Ok((u64::BITS - max_value.leading_zeros()).max(1) as u64)
+    }
+
+    fn child_candidates(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+        compression: Option<CompressionConfig>,
+        use_child_bitpacking: bool,
+    ) -> Result<Vec<RleChildCandidate>> {
+        #[cfg(not(feature = "bitpacking"))]
+        let _ = use_child_bitpacking;
+        let mut candidates = vec![Self::flat_child_candidate(
+            buffers,
+            chunks,
+            buffer_index,
+            bits_per_value,
+        )];
+        if let Some(compression) = compression
+            && let Some(candidate) = Self::general_child_candidate(
+                buffers,
+                chunks,
+                buffer_index,
+                bits_per_value,
+                compression,
+            )?
+        {
+            candidates.push(candidate);
+        }
+        #[cfg(feature = "bitpacking")]
+        {
+            if use_child_bitpacking
+                && let Some(candidate) =
+                    Self::bitpacked_child_candidate(buffers, chunks, buffer_index, bits_per_value)?
+            {
+                candidates.push(candidate);
+            }
+        }
+        Ok(candidates)
+    }
+
+    fn select_child_candidates(
+        values: Vec<RleChildCandidate>,
+        run_lengths: Vec<RleChildCandidate>,
+    ) -> (RleChildCandidate, RleChildCandidate) {
+        let mut best: Option<(usize, usize, usize)> = None;
+        for (value_idx, value) in values.iter().enumerate() {
+            for (length_idx, length) in run_lengths.iter().enumerate() {
+                if value.requires_num_values && length.requires_num_values {
+                    continue;
+                }
+                let size = value.size + length.size;
+                if best.is_none_or(|(_, _, best_size)| size < best_size) {
+                    best = Some((value_idx, length_idx, size));
+                }
+            }
+        }
+        let (value_idx, length_idx, _) =
+            best.expect("flat RLE child candidates should always be selectable");
+        (values[value_idx].clone(), run_lengths[length_idx].clone())
+    }
+
+    pub(crate) fn selected_payload_size(&self, data: &FixedWidthDataBlock) -> Result<u128> {
+        let (all_buffers, chunks) =
+            self.encode_data(&data.data, data.num_values, data.bits_per_value)?;
+        if all_buffers.is_empty() {
+            return Ok(0);
+        }
+
+        let values_candidates = Self::child_candidates(
+            &all_buffers,
+            &chunks,
+            0,
+            data.bits_per_value,
+            self.values_compression,
+            self.use_child_bitpacking,
+        )?;
+        let run_lengths_candidates = Self::child_candidates(
+            &all_buffers,
+            &chunks,
+            1,
+            self.run_length_width.bits_per_value(),
+            self.run_lengths_compression,
+            self.use_child_bitpacking,
+        )?;
+        let (values, run_lengths) =
+            Self::select_child_candidates(values_candidates, run_lengths_candidates);
+        Ok((values.size as u128).saturating_add(run_lengths.size as u128))
+    }
+}
+
+impl RleChildCandidate {
+    fn with_size_from_data(mut self) -> Self {
+        self.size = self.data.len();
+        self
+    }
 }
 
 impl MiniBlockCompressor for RleEncoder {
@@ -689,17 +1044,53 @@ impl MiniBlockCompressor for RleEncoder {
 
                 let (all_buffers, chunks) =
                     self.encode_data(&fixed_width.data, num_values, bits_per_value)?;
+                if all_buffers.is_empty() {
+                    let compressed = MiniBlockCompressed {
+                        data: all_buffers,
+                        chunks,
+                        num_values,
+                    };
+                    let encoding = ProtobufUtils21::rle(
+                        ProtobufUtils21::flat(bits_per_value, None),
+                        ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None),
+                    );
+                    return Ok((compressed, encoding));
+                }
+
+                let values_candidates = Self::child_candidates(
+                    &all_buffers,
+                    &chunks,
+                    0,
+                    bits_per_value,
+                    self.values_compression,
+                    self.use_child_bitpacking,
+                )?;
+                let run_lengths_candidates = Self::child_candidates(
+                    &all_buffers,
+                    &chunks,
+                    1,
+                    self.run_length_width.bits_per_value(),
+                    self.run_lengths_compression,
+                    self.use_child_bitpacking,
+                )?;
+                let (values, run_lengths) =
+                    Self::select_child_candidates(values_candidates, run_lengths_candidates);
+                let chunks = chunks
+                    .into_iter()
+                    .enumerate()
+                    .map(|(idx, chunk)| MiniBlockChunk {
+                        buffer_sizes: vec![values.chunk_sizes[idx], run_lengths.chunk_sizes[idx]],
+                        log_num_values: chunk.log_num_values,
+                    })
+                    .collect();
 
                 let compressed = MiniBlockCompressed {
-                    data: all_buffers,
+                    data: vec![values.data, run_lengths.data],
                     chunks,
                     num_values,
                 };
 
-                let encoding = ProtobufUtils21::rle(
-                    ProtobufUtils21::flat(bits_per_value, None),
-                    ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None),
-                );
+                let encoding = ProtobufUtils21::rle(values.encoding, run_lengths.encoding);
 
                 Ok((compressed, encoding))
             }
@@ -741,6 +1132,125 @@ impl BlockCompressor for RleEncoder {
 pub struct RleDecompressor {
     bits_per_value: u64,
     run_length_width: RunLengthWidth,
+    values: RleChildDecompressor,
+    run_lengths: RleChildDecompressor,
+}
+
+#[derive(Debug)]
+pub(crate) struct RleChildDecompressor {
+    bits_per_value: u64,
+    inner: RleChildDecompressorInner,
+}
+
+#[derive(Debug)]
+enum RleChildDecompressorInner {
+    Flat,
+    Block {
+        decompressor: Box<dyn BlockDecompressor>,
+        requires_num_values: bool,
+    },
+}
+
+impl RleChildDecompressor {
+    pub(crate) fn flat(bits_per_value: u64) -> Self {
+        Self {
+            bits_per_value,
+            inner: RleChildDecompressorInner::Flat,
+        }
+    }
+
+    pub(crate) fn block(
+        bits_per_value: u64,
+        decompressor: Box<dyn BlockDecompressor>,
+        requires_num_values: bool,
+    ) -> Self {
+        Self {
+            bits_per_value,
+            inner: RleChildDecompressorInner::Block {
+                decompressor,
+                requires_num_values,
+            },
+        }
+    }
+
+    pub(crate) fn bits_per_value(&self) -> u64 {
+        self.bits_per_value
+    }
+
+    pub(crate) fn requires_num_values(&self) -> bool {
+        match &self.inner {
+            RleChildDecompressorInner::Flat => false,
+            RleChildDecompressorInner::Block {
+                requires_num_values,
+                ..
+            } => *requires_num_values,
+        }
+    }
+
+    pub(crate) fn is_identity(&self) -> bool {
+        matches!(self.inner, RleChildDecompressorInner::Flat)
+    }
+
+    fn decode(
+        &self,
+        data: LanceBuffer,
+        num_values: Option<u64>,
+        label: &str,
+    ) -> Result<LanceBuffer> {
+        match &self.inner {
+            RleChildDecompressorInner::Flat => Ok(data),
+            RleChildDecompressorInner::Block {
+                decompressor,
+                requires_num_values,
+            } => {
+                let num_values = if *requires_num_values {
+                    num_values.ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!("RLE {label} child compression requires the run count").into(),
+                        )
+                    })?
+                } else {
+                    num_values.unwrap_or(0)
+                };
+                let decoded = decompressor.decompress(data, num_values)?;
+                self.extract_fixed_width(decoded, num_values, label)
+            }
+        }
+    }
+
+    fn extract_fixed_width(
+        &self,
+        data: DataBlock,
+        expected_num_values: u64,
+        label: &str,
+    ) -> Result<LanceBuffer> {
+        match data {
+            DataBlock::FixedWidth(block) => {
+                if block.bits_per_value != self.bits_per_value {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE {label} child decoded {}-bit values, expected {}",
+                            block.bits_per_value, self.bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                if expected_num_values != 0 && block.num_values != expected_num_values {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE {label} child decoded {} values, expected {}",
+                            block.num_values, expected_num_values
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(block.data)
+            }
+            _ => Err(Error::invalid_input_source(
+                format!("RLE {label} child decoded to a non fixed-width block").into(),
+            )),
+        }
+    }
 }
 
 impl RleDecompressor {
@@ -748,6 +1258,8 @@ impl RleDecompressor {
         Self {
             bits_per_value,
             run_length_width: RunLengthWidth::U8,
+            values: RleChildDecompressor::flat(bits_per_value),
+            run_lengths: RleChildDecompressor::flat(RunLengthWidth::U8.bits_per_value()),
         }
     }
 
@@ -758,10 +1270,31 @@ impl RleDecompressor {
         Self {
             bits_per_value,
             run_length_width,
+            values: RleChildDecompressor::flat(bits_per_value),
+            run_lengths: RleChildDecompressor::flat(run_length_width.bits_per_value()),
+        }
+    }
+
+    pub(crate) fn with_child_decompressors(
+        bits_per_value: u64,
+        run_length_width: RunLengthWidth,
+        values: RleChildDecompressor,
+        run_lengths: RleChildDecompressor,
+    ) -> Self {
+        Self {
+            bits_per_value,
+            run_length_width,
+            values,
+            run_lengths,
         }
     }
 
-    fn decode_data(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
+    fn decode_data(
+        &self,
+        data: Vec<LanceBuffer>,
+        num_values: u64,
+        clamp_overflow: bool,
+    ) -> Result<DataBlock> {
         if num_values == 0 {
             return Ok(DataBlock::FixedWidth(FixedWidthDataBlock {
                 bits_per_value: self.bits_per_value,
@@ -781,14 +1314,37 @@ impl RleDecompressor {
             ));
         }
 
-        let values_buffer = &data[0];
-        let lengths_buffer = &data[1];
+        let mut data_iter = data.into_iter();
+        let values_buffer = data_iter.next().unwrap();
+        let lengths_buffer = data_iter.next().unwrap();
+        let (values_buffer, lengths_buffer) =
+            self.decode_child_buffers(values_buffer, lengths_buffer)?;
 
         let decoded_data = match self.bits_per_value {
-            8 => self.decode_generic::<u8>(values_buffer, lengths_buffer, num_values)?,
-            16 => self.decode_generic::<u16>(values_buffer, lengths_buffer, num_values)?,
-            32 => self.decode_generic::<u32>(values_buffer, lengths_buffer, num_values)?,
-            64 => self.decode_generic::<u64>(values_buffer, lengths_buffer, num_values)?,
+            8 => self.decode_generic::<u8>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            16 => self.decode_generic::<u16>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            32 => self.decode_generic::<u32>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            64 => self.decode_generic::<u64>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
             _ => {
                 return Err(Error::invalid_input_source(
                     format!(
@@ -808,11 +1364,74 @@ impl RleDecompressor {
         }))
     }
 
+    fn decode_child_buffers(
+        &self,
+        values_buffer: LanceBuffer,
+        lengths_buffer: LanceBuffer,
+    ) -> Result<(LanceBuffer, LanceBuffer)> {
+        let values_requires_num_runs = self.values.requires_num_values();
+        let lengths_requires_num_runs = self.run_lengths.requires_num_values();
+        if values_requires_num_runs && lengths_requires_num_runs {
+            return Err(Error::invalid_input_source(
+                "RLE values and run lengths child compression both require the run count".into(),
+            ));
+        }
+
+        if values_requires_num_runs {
+            let lengths_buffer = self
+                .run_lengths
+                .decode(lengths_buffer, None, "run lengths")?;
+            let num_runs = Self::num_child_values(
+                &lengths_buffer,
+                self.run_lengths.bits_per_value(),
+                "run lengths",
+            )?;
+            let values_buffer = self
+                .values
+                .decode(values_buffer, Some(num_runs), "values")?;
+            Ok((values_buffer, lengths_buffer))
+        } else if lengths_requires_num_runs {
+            let values_buffer = self.values.decode(values_buffer, None, "values")?;
+            let num_runs =
+                Self::num_child_values(&values_buffer, self.values.bits_per_value(), "values")?;
+            let lengths_buffer =
+                self.run_lengths
+                    .decode(lengths_buffer, Some(num_runs), "run lengths")?;
+            Ok((values_buffer, lengths_buffer))
+        } else {
+            let values_buffer = self.values.decode(values_buffer, None, "values")?;
+            let lengths_buffer = self
+                .run_lengths
+                .decode(lengths_buffer, None, "run lengths")?;
+            Ok((values_buffer, lengths_buffer))
+        }
+    }
+
+    fn num_child_values(buffer: &LanceBuffer, bits_per_value: u64, label: &str) -> Result<u64> {
+        let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE {label} child bit width is too large: {bits_per_value}").into(),
+            )
+        })?;
+        if bytes_per_value == 0 || !buffer.len().is_multiple_of(bytes_per_value) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "RLE {label} child decoded to {} bytes, not divisible by {}",
+                    buffer.len(),
+                    bytes_per_value
+                )
+                .into(),
+            ));
+        }
+        Ok((buffer.len() / bytes_per_value) as u64)
+    }
+
     fn decode_generic<T>(
         &self,
         values_buffer: &LanceBuffer,
         lengths_buffer: &LanceBuffer,
         num_values: u64,
+        clamp_overflow: bool,
     ) -> Result<LanceBuffer>
     where
         T: bytemuck::Pod + Copy + std::fmt::Debug + ArrowNativeType,
@@ -865,8 +1484,21 @@ impl RleDecompressor {
                 format!("RLE num_values does not fit in usize: {num_values}").into(),
             )
         })?;
-        let mut decoded_value_count = 0usize;
-        for length_bytes in lengths.chunks_exact(length_size) {
+        // Legacy miniblock encoders rolled back to a power-of-2 checkpoint after a run
+        // had already crossed it, so a chunk's run lengths can sum past its declared
+        // value count (the excess values are re-encoded at the start of the next chunk).
+        // The pre-run-length-width decoder truncated the excess, so miniblock decoding
+        // clamps rather than rejects to keep those files readable. Block payloads never
+        // legitimately overflow, so they decode strictly.
+        let mut decoded: Vec<T> = Vec::new();
+        decoded
+            .try_reserve_exact(expected_value_count)
+            .map_err(|_| {
+                Error::invalid_input_source(
+                    format!("RLE decoding cannot allocate {expected_value_count} values").into(),
+                )
+            })?;
+        for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) {
             let length = self.run_length_width.read_length(length_bytes);
             if length == 0 {
                 return Err(Error::invalid_input_source(
@@ -878,36 +1510,35 @@ impl RleDecompressor {
                     format!("RLE run length does not fit in usize: {length}").into(),
                 )
             })?;
-            decoded_value_count = decoded_value_count.checked_add(length).ok_or_else(|| {
-                Error::invalid_input_source("RLE run length sum overflowed usize".into())
-            })?;
-            if decoded_value_count > expected_value_count {
-                return Err(Error::invalid_input_source(
-                    format!(
-                        "RLE decoding overflowed expected value count: produced at least {}, expected {}",
-                        decoded_value_count, expected_value_count
-                    )
-                    .into(),
-                ));
+            let remaining = expected_value_count - decoded.len();
+            if length > remaining {
+                if !clamp_overflow {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE decoding overflowed expected value count: produced at least {}, expected {}",
+                            decoded.len() + length,
+                            expected_value_count
+                        )
+                        .into(),
+                    ));
+                }
+                decoded.resize(expected_value_count, *value);
+                break;
             }
+            decoded.resize(decoded.len() + length, *value);
         }
 
-        if decoded_value_count != expected_value_count {
+        if decoded.len() != expected_value_count {
             return Err(Error::invalid_input_source(
                 format!(
                     "RLE decoding produced {} values, expected {}",
-                    decoded_value_count, expected_value_count
+                    decoded.len(),
+                    expected_value_count
                 )
                 .into(),
             ));
         }
 
-        let mut decoded: Vec<T> = Vec::with_capacity(expected_value_count);
-        for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) {
-            let length = self.run_length_width.read_length(length_bytes) as usize;
-            decoded.resize(decoded.len() + length, *value);
-        }
-
         trace!(
             "RLE decoded {} {} values",
             num_values,
@@ -919,7 +1550,7 @@ impl RleDecompressor {
 
 impl MiniBlockDecompressor for RleDecompressor {
     fn decompress(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
-        self.decode_data(data, num_values)
+        self.decode_data(data, num_values, true)
     }
 }
 
@@ -956,17 +1587,24 @@ impl BlockDecompressor for RleDecompressor {
         let values_buffer = data.slice_with_length(values_start, values_size);
         let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start);
 
-        self.decode_data(vec![values_buffer, lengths_buffer], num_values)
+        self.decode_data(vec![values_buffer, lengths_buffer], num_values, false)
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy};
     use crate::data::DataBlock;
     use crate::encodings::logical::primitive::miniblock::MAX_MINIBLOCK_VALUES;
-    use crate::{buffer::LanceBuffer, compression::BlockDecompressor};
+    use crate::encodings::physical::block::{CompressionConfig, CompressionScheme};
+    use crate::{
+        buffer::LanceBuffer,
+        compression::{BlockCompressor, BlockDecompressor},
+    };
     use arrow_array::Int32Array;
+    use rstest::rstest;
+
     // ========== Core Functionality Tests ==========
 
     #[test]
@@ -1046,6 +1684,323 @@ mod tests {
         }
     }
 
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_values_child() {
+        let compression = test_general_compression();
+        let encoder =
+            RleEncoder::with_child_encoding(RunLengthWidth::U8, Some(compression), None, false);
+        let array = Int32Array::from(repeating_runs(1024, 4));
+        let (compressed, encoding) =
+            MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded =
+            MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4)
+                .unwrap();
+        assert_decoded_i32_eq(decoded, &repeating_runs(1024, 4));
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_run_lengths_child() {
+        let compression = test_general_compression();
+        let encoder =
+            RleEncoder::with_child_encoding(RunLengthWidth::U8, None, Some(compression), false);
+        let expected = repeating_runs(1024, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded =
+            MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4)
+                .unwrap();
+        assert_decoded_i32_eq(decoded, &expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacked_run_lengths_child() {
+        use crate::encodings::physical::bitpacking::OutOfLineBitpacking;
+
+        let expected = repeating_runs(1024, 4);
+        let (compressed, _) = MiniBlockCompressor::compress(
+            &RleEncoder::new(),
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+        let run_lengths = compressed.data[1].clone();
+        let num_runs = run_lengths.len() as u64;
+        let run_lengths_block = DataBlock::FixedWidth(FixedWidthDataBlock {
+            bits_per_value: 8,
+            data: run_lengths,
+            num_values: num_runs,
+            block_info: BlockInfo::default(),
+        });
+        let bitpacked_run_lengths =
+            BlockCompressor::compress(&OutOfLineBitpacking::new(3, 8), run_lengths_block).unwrap();
+        let encoding = ProtobufUtils21::rle(
+            ProtobufUtils21::flat(32, None),
+            ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)),
+        );
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded = MiniBlockDecompressor::decompress(
+            decompressor.as_ref(),
+            vec![compressed.data[0].clone(), bitpacked_run_lengths],
+            expected.len() as u64,
+        )
+        .unwrap();
+        assert_decoded_i32_eq(decoded, &expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_rejects_two_count_dependent_child_encodings() {
+        let encoding = ProtobufUtils21::rle(
+            ProtobufUtils21::out_of_line_bitpacking(32, ProtobufUtils21::flat(3, None)),
+            ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)),
+        );
+
+        let err = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("cannot both require the run count")
+        );
+    }
+
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_general_compression() -> CompressionConfig {
+        if cfg!(feature = "zstd") {
+            CompressionConfig::new(CompressionScheme::Zstd, Some(3))
+        } else {
+            CompressionConfig::new(CompressionScheme::Lz4, None)
+        }
+    }
+
+    fn repeating_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        for run in 0..num_runs {
+            values.extend(std::iter::repeat_n((run % 8) as i32, run_length));
+        }
+        values
+    }
+
+    fn expect_rle(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle {
+        match encoding.compression.as_ref().unwrap() {
+            crate::format::pb21::compressive_encoding::Compression::Rle(rle) => rle,
+            other => panic!("expected RLE encoding, got {other:?}"),
+        }
+    }
+
+    fn assert_decoded_i32_eq(decoded: DataBlock, expected: &[i32]) {
+        match decoded {
+            DataBlock::FixedWidth(block) => {
+                let values = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(values.as_ref(), expected);
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_children_multiple_chunks() {
+        let compression = test_general_compression();
+        let encoder = RleEncoder::with_child_encoding(
+            RunLengthWidth::U8,
+            Some(compression),
+            Some(compression),
+            false,
+        );
+        let expected = repeating_runs(8192, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        assert!(compressed.chunks.len() > 1);
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacks_values_child_when_smaller() {
+        let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true);
+        let expected = monotonic_runs(2048, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacks_run_lengths_when_values_do_not_shrink() {
+        let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true);
+        let expected = high_entropy_runs(2048, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    fn decompress_i32_chunks(
+        compressed: &MiniBlockCompressed,
+        encoding: &CompressiveEncoding,
+    ) -> Vec<i32> {
+        let strategy = DefaultDecompressionStrategy::default();
+        let decompressor = strategy
+            .create_miniblock_decompressor(encoding, &strategy)
+            .unwrap();
+        let mut offsets = vec![0usize; compressed.data.len()];
+        let mut values_processed = 0u64;
+        let mut decoded_values = Vec::new();
+
+        for chunk in &compressed.chunks {
+            let chunk_values = chunk.num_values(values_processed, compressed.num_values);
+            let mut chunk_buffers = Vec::with_capacity(chunk.buffer_sizes.len());
+            for (idx, size) in chunk.buffer_sizes.iter().enumerate() {
+                let size = *size as usize;
+                chunk_buffers.push(compressed.data[idx].slice_with_length(offsets[idx], size));
+                offsets[idx] += size;
+            }
+
+            let decoded = decompressor
+                .decompress(chunk_buffers, chunk_values)
+                .unwrap();
+            match decoded {
+                DataBlock::FixedWidth(block) => {
+                    let values = block.data.borrow_to_typed_slice::<i32>();
+                    decoded_values.extend_from_slice(values.as_ref());
+                }
+                _ => panic!("Expected FixedWidth block"),
+            }
+            values_processed += chunk_values;
+        }
+
+        assert_eq!(values_processed, compressed.num_values);
+        decoded_values
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn monotonic_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        for run in 0..num_runs {
+            values.extend(std::iter::repeat_n(run as i32, run_length));
+        }
+        values
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn high_entropy_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        let mut state = 7u64;
+        for _ in 0..num_runs {
+            state = state
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407);
+            values.extend(std::iter::repeat_n((state >> 32) as i32, run_length));
+        }
+        values
+    }
+
     #[test]
     fn test_select_run_length_width_prefers_u16_for_long_runs() {
         let mut entries = [0u64; 3];
@@ -1139,8 +2094,73 @@ mod tests {
         }
     }
 
+    #[rstest]
+    #[case::u8_lengths(RunLengthWidth::U8)]
+    #[case::u16_lengths(RunLengthWidth::U16)]
+    #[case::u32_lengths(RunLengthWidth::U32)]
+    fn test_miniblock_chunk_counts_match_encoded_runs(#[case] run_length_width: RunLengthWidth) {
+        // This pattern crosses the 2,048-value boundary in the middle of a two-value run.
+        let levels = (0..4098)
+            .map(|index| if index % 3 == 0 { 1u16 } else { 0u16 })
+            .collect::<Vec<_>>();
+        let num_values = levels.len() as u64;
+        let encoder = RleEncoder::with_run_length_width(run_length_width);
+        let (buffers, chunks) = encoder
+            .encode_data(
+                &LanceBuffer::reinterpret_vec(levels),
+                num_values,
+                u16::BITS as u64,
+            )
+            .unwrap();
+
+        assert_eq!(buffers.len(), 2);
+        let bytes_per_length = run_length_width.bytes_per_value();
+        let mut values_offset = 0usize;
+        let mut lengths_offset = 0usize;
+        let mut values_processed = 0u64;
+
+        for chunk in &chunks {
+            let values_size = chunk.buffer_sizes[0] as usize;
+            let lengths_size = chunk.buffer_sizes[1] as usize;
+            let lengths_end = lengths_offset + lengths_size;
+            let chunk_lengths = &buffers[1].as_ref()[lengths_offset..lengths_end];
+            let length_chunks = chunk_lengths.chunks_exact(bytes_per_length);
+            assert!(length_chunks.remainder().is_empty());
+            let num_runs = length_chunks.len();
+            let encoded_values = length_chunks
+                .map(|bytes| run_length_width.read_length(bytes))
+                .sum::<u64>();
+            let declared_values = chunk.num_values(values_processed, num_values);
+
+            assert_eq!(values_size, num_runs * size_of::<u16>());
+            assert_eq!(encoded_values, declared_values);
+
+            values_offset += values_size;
+            lengths_offset = lengths_end;
+            values_processed += declared_values;
+        }
+
+        assert_eq!(values_processed, num_values);
+        assert_eq!(values_offset, buffers[0].len());
+        assert_eq!(lengths_offset, buffers[1].len());
+    }
+
     // ========== Error Handling Tests ==========
 
+    #[test]
+    fn test_encoder_rejects_zero_progress() {
+        let error = RleEncoder::new()
+            .encode_data(&LanceBuffer::empty(), 1, u16::BITS as u64)
+            .unwrap_err();
+
+        assert!(
+            matches!(&error, Error::Internal { .. }),
+            "expected internal error, got: {error:?}"
+        );
+        assert!(error.to_string().contains("made no progress"));
+        assert!(error.to_string().contains("values_remaining=1"));
+    }
+
     #[test]
     fn test_invalid_buffer_count() {
         let decompressor = RleDecompressor::new(32);
@@ -1189,7 +2209,7 @@ mod tests {
     }
 
     #[test]
-    fn test_rle_rejects_underflow_overflow_and_zero_lengths() {
+    fn test_rle_rejects_underflow_and_zero_lengths_and_clamps_overflow() {
         let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
         let value = LanceBuffer::from(1i32.to_le_bytes().to_vec());
 
@@ -1212,12 +2232,15 @@ mod tests {
             ],
             5,
         )
-        .unwrap_err();
-        assert!(
-            overflow
-                .to_string()
-                .contains("overflowed expected value count")
-        );
+        .unwrap();
+        match overflow {
+            DataBlock::FixedWidth(block) => {
+                assert_eq!(block.num_values, 5);
+                let decoded = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(decoded.as_ref(), &[1i32; 5]);
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
 
         let zero = MiniBlockDecompressor::decompress(
             &decompressor,
@@ -1228,6 +2251,60 @@ mod tests {
         assert!(zero.to_string().contains("zero run length"));
     }
 
+    #[test]
+    fn test_block_rle_rejects_overflow() {
+        // Block payloads have no chunk boundaries, so run lengths summing past
+        // num_values can only be corruption and must stay a hard error.
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let values = 1i32.to_le_bytes();
+        let lengths = 6u16.to_le_bytes();
+        let mut payload = Vec::new();
+        payload.extend_from_slice(&(values.len() as u64).to_le_bytes());
+        payload.extend_from_slice(&values);
+        payload.extend_from_slice(&lengths);
+
+        let error = BlockDecompressor::decompress(&decompressor, LanceBuffer::from(payload), 5)
+            .unwrap_err();
+        assert!(matches!(&error, Error::InvalidInput { .. }));
+        assert!(
+            error
+                .to_string()
+                .contains("overflowed expected value count")
+        );
+    }
+
+    #[test]
+    fn test_rle_truncates_legacy_chunk_boundary_overflow() {
+        // Legacy encoders emitted chunks declaring 2048 values whose final run crossed
+        // the checkpoint boundary (e.g. run lengths summing to 2080); the excess values
+        // are duplicated at the start of the next chunk and must be ignored here.
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let mut values = Vec::new();
+        values.extend_from_slice(&7i32.to_le_bytes());
+        values.extend_from_slice(&8i32.to_le_bytes());
+        let mut lengths = Vec::new();
+        lengths.extend_from_slice(&2000u16.to_le_bytes());
+        lengths.extend_from_slice(&80u16.to_le_bytes());
+
+        let decoded = MiniBlockDecompressor::decompress(
+            &decompressor,
+            vec![LanceBuffer::from(values), LanceBuffer::from(lengths)],
+            2048,
+        )
+        .unwrap();
+        match decoded {
+            DataBlock::FixedWidth(block) => {
+                assert_eq!(block.num_values, 2048);
+                let decoded = block.data.borrow_to_typed_slice::<i32>();
+                let decoded = decoded.as_ref();
+                assert_eq!(decoded.len(), 2048);
+                assert!(decoded[..2000].iter().all(|&v| v == 7));
+                assert!(decoded[2000..].iter().all(|&v| v == 8));
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+    }
+
     #[test]
     fn test_empty_data_handling() {
         let encoder = RleEncoder::new();
@@ -1566,6 +2643,30 @@ mod tests {
         ); // 20% variety
         let arr = Arc::new(Int32Array::from(values)) as Arc<dyn Array>;
         check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await;
+
+        #[cfg(any(feature = "lz4", feature = "zstd"))]
+        {
+            let mut metadata = HashMap::new();
+            metadata.insert(
+                "lance-encoding:rle-threshold".to_string(),
+                "0.8".to_string(),
+            );
+            metadata.insert("lance-encoding:bss".to_string(), "off".to_string());
+            metadata.insert(
+                "lance-encoding:compression".to_string(),
+                if cfg!(feature = "zstd") {
+                    "zstd".to_string()
+                } else {
+                    "lz4".to_string()
+                },
+            );
+            let mut values = Vec::with_capacity(2048 * 4);
+            for run in 0..2048 {
+                values.extend(std::iter::repeat_n(i32::MIN + (run % 8), 4));
+            }
+            let arr = Arc::new(Int32Array::from(values)) as Arc<dyn Array>;
+            check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await;
+        }
     }
 
     /// Generator that produces repetitive patterns suitable for RLE
diff --git a/rust/lance-encoding/src/encodings/physical/value.rs b/rust/lance-encoding/src/encodings/physical/value.rs
index c49bbd3efbd..606f49b699a 100644
--- a/rust/lance-encoding/src/encodings/physical/value.rs
+++ b/rust/lance-encoding/src/encodings/physical/value.rs
@@ -27,7 +27,7 @@ pub struct ValueEncoder {}
 
 impl ValueEncoder {
     /// Use the largest chunk we can smaller than 4KiB
-    fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> (u64, u64) {
+    fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> Result<(u64, u64)> {
         let mut size_bytes = 2 * bytes_per_word;
         let (mut log_num_vals, mut num_vals) = match values_per_word {
             1 => (1, 2),
@@ -35,8 +35,14 @@ impl ValueEncoder {
             _ => unreachable!(),
         };
 
-        // If the type is so wide that we can't even fit 2 values we shouldn't be here
-        assert!(size_bytes < MAX_MINIBLOCK_BYTES);
+        if size_bytes >= MAX_MINIBLOCK_BYTES {
+            let num_values = 2 * values_per_word;
+            return Err(Error::invalid_input(format!(
+                "Value is too wide for miniblock encoding: {} values require {} bytes but a \
+                 miniblock chunk is limited to {} bytes.",
+                num_values, size_bytes, MAX_MINIBLOCK_BYTES
+            )));
+        }
 
         while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= *MAX_MINIBLOCK_VALUES {
             log_num_vals += 1;
@@ -44,10 +50,10 @@ impl ValueEncoder {
             num_vals *= 2;
         }
 
-        (log_num_vals, num_vals)
+        Ok((log_num_vals, num_vals))
     }
 
-    fn chunk_data(data: FixedWidthDataBlock) -> MiniBlockCompressed {
+    fn chunk_data(data: FixedWidthDataBlock) -> Result<MiniBlockCompressed> {
         // Usually there are X bytes per value.  However, when working with boolean
         // or FSL<boolean> we might have some number of bits per value that isn't
         // divisible by 8.  In this case, to avoid chunking in the middle of a byte
@@ -60,7 +66,7 @@ impl ValueEncoder {
 
         // Aim for 4KiB chunks
         let (log_vals_per_chunk, vals_per_chunk) =
-            Self::find_log_vals_per_chunk(bytes_per_word, values_per_word);
+            Self::find_log_vals_per_chunk(bytes_per_word, values_per_word)?;
         let num_chunks = bit_util::ceil(data.num_values as usize, vals_per_chunk as usize);
         debug_assert_eq!(vals_per_chunk % values_per_word, 0);
         let bytes_per_chunk = bytes_per_word * (vals_per_chunk / values_per_word);
@@ -99,11 +105,11 @@ impl ValueEncoder {
 
         debug_assert_eq!(chunks.len(), num_chunks);
 
-        MiniBlockCompressed {
+        Ok(MiniBlockCompressed {
             chunks,
             data: vec![data_buffer],
             num_values: data.num_values,
-        }
+        })
     }
 }
 
@@ -177,7 +183,7 @@ impl ValueEncoder {
         data: FixedWidthDataBlock,
         layers: Vec<MiniblockFslLayer>,
         num_rows: u64,
-    ) -> (MiniBlockCompressed, CompressiveEncoding) {
+    ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> {
         // Count size to calculate rows per chunk
         let mut ceil_bytes_validity = 0;
         let mut cum_dim = 1;
@@ -198,7 +204,7 @@ impl ValueEncoder {
         };
         let est_bytes_per_word = (ceil_bytes_validity * vals_per_word) + cum_bytes_per_word;
         let (log_rows_per_chunk, rows_per_chunk) =
-            Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word);
+            Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word)?;
 
         let num_chunks = num_rows.div_ceil(rows_per_chunk) as usize;
 
@@ -258,17 +264,17 @@ impl ValueEncoder {
             .chain(std::iter::once(data.data))
             .collect::<Vec<_>>();
 
-        (
+        Ok((
             MiniBlockCompressed {
                 chunks,
                 data: buffers,
                 num_values: num_rows,
             },
             encoding,
-        )
+        ))
     }
 
-    fn miniblock_fsl(data: DataBlock) -> (MiniBlockCompressed, CompressiveEncoding) {
+    fn miniblock_fsl(data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> {
         let num_rows = data.num_values();
         let fsl = data.as_fixed_size_list().unwrap();
         let mut layers = Vec::new();
@@ -469,9 +475,9 @@ impl MiniBlockCompressor for ValueEncoder {
         match chunk {
             DataBlock::FixedWidth(fixed_width) => {
                 let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None);
-                Ok((Self::chunk_data(fixed_width), encoding))
+                Ok((Self::chunk_data(fixed_width)?, encoding))
             }
-            DataBlock::FixedSizeList(_) => Ok(Self::miniblock_fsl(chunk)),
+            DataBlock::FixedSizeList(_) => Self::miniblock_fsl(chunk),
             _ => Err(Error::invalid_input_source(
                 format!(
                     "Cannot compress a data block of type {} with ValueEncoder",
@@ -989,6 +995,59 @@ mod tests {
         assert_eq!(decompressed.as_ref(), &sample_list);
     }
 
+    fn wide_fixed_size_binary() -> ArrayRef {
+        let wide_value = vec![0xABu8; 5000];
+        Arc::new(
+            arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                std::iter::repeat_n(Some(wide_value.as_slice()), 4),
+                5000,
+            )
+            .unwrap(),
+        )
+    }
+
+    fn wide_fixed_size_list_bool() -> ArrayRef {
+        // A wide FSL<Boolean> is sub-byte, so it chunks eight values per word and the
+        // smallest unit is 16 values rather than 2.
+        let dimension = 4095;
+        let values = arrow_array::BooleanArray::from(vec![false; dimension * 2]);
+        let field = Arc::new(Field::new("item", DataType::Boolean, true));
+        Arc::new(FixedSizeListArray::new(
+            field,
+            dimension as i32,
+            Arc::new(values),
+            None,
+        ))
+    }
+
+    #[rstest::rstest]
+    #[case::fixed_size_binary(wide_fixed_size_binary(), 2)]
+    #[case::fixed_size_list_bool(wide_fixed_size_list_bool(), 16)]
+    fn test_wide_value_miniblock_returns_error(
+        #[case] array: ArrayRef,
+        #[case] expected_min_values: u64,
+    ) {
+        let starting_data = DataBlock::from_array(array);
+
+        let encoder = ValueEncoder::default();
+        let result = MiniBlockCompressor::compress(&encoder, starting_data);
+
+        let err = result.expect_err("wide values should not be encodable as miniblock");
+        assert!(
+            matches!(err, lance_core::Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+        let msg = err.to_string();
+        assert!(
+            msg.contains("too wide for miniblock encoding"),
+            "unexpected error message: {msg}"
+        );
+        assert!(
+            msg.contains(&format!("{expected_min_values} values require")),
+            "unexpected error message: {msg}"
+        );
+    }
+
     #[test]
     fn test_fsl_value_compression_per_value() {
         let sample_list = create_simple_fsl();
diff --git a/rust/lance-encoding/src/previous/encodings/logical/blob.rs b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
index 13fa3b346cb..20f6afc5b36 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/blob.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
@@ -318,7 +318,7 @@ impl BlobFieldEncoder {
             .nulls()
             .cloned()
             .unwrap_or(NullBuffer::new_valid(binarray.len()));
-        for (w, is_valid) in binarray.value_offsets().windows(2).zip(nulls.into_iter()) {
+        for (w, is_valid) in binarray.value_offsets().windows(2).zip(&nulls) {
             if is_valid {
                 let start = w[0] as u64;
                 let end = w[1] as u64;
diff --git a/rust/lance-encoding/src/repdef.rs b/rust/lance-encoding/src/repdef.rs
index b418b906de8..bb5d9cca3af 100644
--- a/rust/lance-encoding/src/repdef.rs
+++ b/rust/lance-encoding/src/repdef.rs
@@ -118,13 +118,16 @@ use arrow_buffer::{
 };
 use lance_core::{Error, Result, utils::bit::log_2_ceil};
 
-use crate::buffer::LanceBuffer;
+use crate::{
+    buffer::LanceBuffer,
+    encodings::logical::primitive::sparse::{SparseStructuralPlan, SparseStructuralUnraveler},
+};
 
 pub type LevelBuffer = Vec<u16>;
 
-/// A contiguous top-level-row range that can be encoded as one structural page.
+/// A top-level-row range whose dense rep/def stream fits one mini-block page.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct StructuralPageSplit {
+pub(crate) struct MiniBlockRepDefSplit {
     /// Top-level row offset, relative to the original unsplit page.
     pub(crate) row_start: u64,
     /// Number of top-level rows in this split.
@@ -137,15 +140,15 @@ pub(crate) struct StructuralPageSplit {
     pub(crate) num_values: u64,
 }
 
-/// Planner result for structural page budget handling.
+/// Dense mini-block rep/def budget result for one accumulated page.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum StructuralPagePlan {
-    /// The original page can be encoded as-is.
-    Fits,
-    /// The original page should be split on top-level row boundaries.
-    Split(Vec<StructuralPageSplit>),
-    /// One top-level row is larger than the requested structural page budget.
-    UnsplittableOverBudget(u64),
+pub(crate) enum MiniBlockRepDefBudget {
+    /// The dense rep/def stream fits one mini-block structural page.
+    WithinBudget,
+    /// The dense rep/def stream fits after splitting on top-level row boundaries.
+    RequiresPageSplit(Vec<MiniBlockRepDefSplit>),
+    /// A single top-level row has this many rep/def levels and exceeds the budget.
+    SingleRowOverBudget(u64),
 }
 
 // As we build def levels we add this to special values to indicate that they
@@ -199,6 +202,143 @@ enum RawRepDef {
     Fsl(FslDesc),
 }
 
+/// A normalized Arrow structural layer shared by dense and sparse serializers.
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum NormalizedStructuralLayer<'a> {
+    List {
+        offsets: &'a [i64],
+        validity: Option<&'a BooleanBuffer>,
+        num_slots: usize,
+    },
+    Validity {
+        validity: Option<&'a BooleanBuffer>,
+        num_slots: usize,
+    },
+    FixedSizeList {
+        validity: Option<&'a BooleanBuffer>,
+        dimension: usize,
+        num_slots: usize,
+    },
+}
+
+/// Structural layers concatenated across input batches exactly once.
+///
+/// Dense rep/def serialization and sparse metadata planning both consume this
+/// representation so the Arrow nesting is not independently reconstructed.
+#[derive(Debug)]
+pub(crate) struct NormalizedStructuralPlan {
+    layers: Vec<RawRepDef>,
+    dense_all_valid: bool,
+}
+
+impl NormalizedStructuralPlan {
+    pub(crate) fn layers(&self) -> impl ExactSizeIterator<Item = NormalizedStructuralLayer<'_>> {
+        self.layers.iter().map(|layer| match layer {
+            RawRepDef::Offsets(OffsetDesc {
+                offsets,
+                validity,
+                num_values,
+                ..
+            }) => NormalizedStructuralLayer::List {
+                offsets,
+                validity: validity.as_ref(),
+                num_slots: *num_values,
+            },
+            RawRepDef::Validity(ValidityDesc {
+                validity,
+                num_values,
+            }) => NormalizedStructuralLayer::Validity {
+                validity: validity.as_ref(),
+                num_slots: *num_values,
+            },
+            RawRepDef::Fsl(FslDesc {
+                validity,
+                dimension,
+                num_values,
+            }) => NormalizedStructuralLayer::FixedSizeList {
+                validity: validity.as_ref(),
+                dimension: *dimension,
+                num_slots: *num_values,
+            },
+        })
+    }
+
+    fn to_serializer(&self) -> (SerializerContext, Option<u64>) {
+        if self.dense_all_valid {
+            let def_meaning = self
+                .layers
+                .iter()
+                .map(|_| DefinitionInterpretation::AllValidItem)
+                .collect::<Vec<_>>();
+            return (
+                SerializerContext {
+                    def_meaning,
+                    rep_levels: LevelBuffer::default(),
+                    spare_rep: LevelBuffer::default(),
+                    def_levels: LevelBuffer::default(),
+                    spare_def: LevelBuffer::default(),
+                    current_rep: 0,
+                    current_def: 0,
+                    current_len: 0,
+                    current_num_specials: 0,
+                    has_fsl: false,
+                },
+                None,
+            );
+        }
+
+        let total_len = self.layers.last().map_or(0, RawRepDef::num_values)
+            + self
+                .layers
+                .iter()
+                .map(RawRepDef::num_specials)
+                .sum::<usize>();
+        let max_rep = self.layers.iter().map(RawRepDef::max_rep).sum::<u16>();
+        let max_def = self.layers.iter().map(RawRepDef::max_def).sum::<u16>();
+        let bits_per_rep = if max_rep > 0 {
+            u64::from(u16::BITS - max_rep.leading_zeros())
+        } else {
+            0
+        };
+        let bits_per_def = if max_def > 0 {
+            u64::from(u16::BITS - max_def.leading_zeros())
+        } else {
+            0
+        };
+        let bits_per_level =
+            (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def);
+
+        let num_layers = self.layers.len();
+        let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def);
+        for layer in &self.layers {
+            match layer {
+                RawRepDef::Validity(def) => context.record_validity(def),
+                RawRepDef::Offsets(rep) => context.record_offsets(rep),
+                RawRepDef::Fsl(fsl) => context.record_fsl(fsl),
+            }
+        }
+        (context, bits_per_level)
+    }
+
+    pub(crate) fn serialize(&self) -> SerializedRepDefs {
+        self.to_serializer().0.build()
+    }
+
+    pub(crate) fn serialize_with_miniblock_repdef_budget(
+        &self,
+        max_levels_for_bits: impl FnOnce(u64) -> u64,
+        num_rows: u64,
+        num_values: u64,
+    ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> {
+        let (context, bits_per_level) = self.to_serializer();
+        context.build_with_miniblock_repdef_budget(
+            bits_per_level.map(max_levels_for_bits),
+            num_rows,
+            num_values,
+        )
+    }
+}
+
 impl RawRepDef {
     // Are there any nulls in this layer
     fn has_nulls(&self) -> bool {
@@ -806,21 +946,20 @@ impl SerializerContext {
         max_levels_per_page: Option<u64>,
         num_rows: u64,
         num_values: u64,
-    ) -> Result<StructuralPagePlan> {
+    ) -> Result<MiniBlockRepDefBudget> {
         // Extremely sparse lists can have many rep/def levels for very few
         // visible leaf values.  If this ratio becomes too skewed then a
-        // miniblock structural chunk can exceed its packed rep/def metadata
-        // budget even though the value buffers are small.  We detect that case
-        // while normalizing special def levels and split the structural page on
-        // top-level row boundaries so each emitted page stays within the
-        // miniblock structural budget.
+        // mini-block rep/def chunk can exceed its packed metadata budget even
+        // though the value buffers are small.  We detect that case while
+        // normalizing special def levels and split on top-level row boundaries
+        // so each emitted dense mini-block page stays within the budget.
         if self.def_levels.is_empty() {
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         if self.rep_levels.is_empty() {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         if self.rep_levels.len() != self.def_levels.len() {
@@ -833,12 +972,12 @@ impl SerializerContext {
 
         let Some(max_levels_per_page) = max_levels_per_page else {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         };
 
         if num_values == 0 {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         let max_schema_rep = def_meaning.iter().filter(|level| level.is_list()).count() as u16;
@@ -847,7 +986,7 @@ impl SerializerContext {
 
         if !should_plan {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         let max_visible_level = max_visible_level.unwrap();
@@ -855,7 +994,7 @@ impl SerializerContext {
         let mut counted_rows = 0u64;
         let mut counted_values = 0u64;
         let mut saw_structural_overhead = false;
-        let mut unsplittable_over_budget = None;
+        let mut single_row_over_budget_levels = None;
 
         let mut current_row_level_start = None;
         let mut current_row_num_values = 0u64;
@@ -876,14 +1015,14 @@ impl SerializerContext {
                 saw_structural_overhead |= row_has_structural_overhead;
 
                 if row_has_structural_overhead && row_num_levels > max_levels_per_page {
-                    unsplittable_over_budget = Some(row_num_levels);
+                    single_row_over_budget_levels = Some(row_num_levels);
                 }
 
                 if current_page_num_rows > 0
                     && (current_page_has_structural_overhead || row_has_structural_overhead)
                     && current_page_num_levels + row_num_levels > max_levels_per_page
                 {
-                    splits.push(StructuralPageSplit {
+                    splits.push(MiniBlockRepDefSplit {
                         row_start: current_page_row_start,
                         num_rows: current_page_num_rows,
                         level_range: current_page_level_start..current_page_level_end,
@@ -966,14 +1105,14 @@ impl SerializerContext {
             )));
         }
         if !saw_structural_overhead {
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
-        if let Some(row_num_levels) = unsplittable_over_budget {
-            return Ok(StructuralPagePlan::UnsplittableOverBudget(row_num_levels));
+        if let Some(row_num_levels) = single_row_over_budget_levels {
+            return Ok(MiniBlockRepDefBudget::SingleRowOverBudget(row_num_levels));
         }
 
         if current_page_num_rows > 0 {
-            splits.push(StructuralPageSplit {
+            splits.push(MiniBlockRepDefSplit {
                 row_start: current_page_row_start,
                 num_rows: current_page_num_rows,
                 level_range: current_page_level_start..current_page_level_end,
@@ -983,9 +1122,9 @@ impl SerializerContext {
         }
 
         if splits.len() > 1 {
-            Ok(StructuralPagePlan::Split(splits))
+            Ok(MiniBlockRepDefBudget::RequiresPageSplit(splits))
         } else {
-            Ok(StructuralPagePlan::Fits)
+            Ok(MiniBlockRepDefBudget::WithinBudget)
         }
     }
 
@@ -1023,12 +1162,12 @@ impl SerializerContext {
         )
     }
 
-    fn build_with_structural_plan(
+    fn build_with_miniblock_repdef_budget(
         mut self,
         max_levels_per_page: Option<u64>,
         num_rows: u64,
         num_values: u64,
-    ) -> Result<(SerializedRepDefs, StructuralPagePlan)> {
+    ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> {
         if self.current_len == 0 {
             return Ok((
                 SerializedRepDefs::new_with_fixed_size_list_levels(
@@ -1037,7 +1176,7 @@ impl SerializerContext {
                     self.def_meaning,
                     self.has_fsl,
                 ),
-                StructuralPagePlan::Fits,
+                MiniBlockRepDefBudget::WithinBudget,
             ));
         }
 
@@ -1046,7 +1185,7 @@ impl SerializerContext {
             .into_iter()
             .rev()
             .collect::<Vec<_>>();
-        let plan = self.normalize_specials_and_plan_splits(
+        let budget = self.normalize_specials_and_plan_splits(
             &def_meaning,
             max_levels_per_page,
             num_rows,
@@ -1071,7 +1210,7 @@ impl SerializerContext {
                 def_meaning,
                 self.has_fsl,
             ),
-            plan,
+            budget,
         ))
     }
 }
@@ -1409,54 +1548,18 @@ impl RepDefBuilder {
     /// Converts the validity / offsets buffers that have been gathered so far
     /// into repetition and definition levels
     pub fn serialize(builders: Vec<Self>) -> SerializedRepDefs {
-        Self::serialize_builders(builders).0.build()
+        Self::normalize(builders).serialize()
     }
 
-    /// Converts gathered structural buffers into rep/def levels and an encode-time plan.
-    pub(crate) fn serialize_with_structural_plan(
-        builders: Vec<Self>,
-        max_levels_for_bits: impl FnOnce(u64) -> u64,
-        num_rows: u64,
-        num_values: u64,
-    ) -> Result<(SerializedRepDefs, StructuralPagePlan)> {
-        let (context, bits_per_level) = Self::serialize_builders(builders);
-        context.build_with_structural_plan(
-            bits_per_level.map(max_levels_for_bits),
-            num_rows,
-            num_values,
-        )
-    }
-
-    fn serialize_builders(builders: Vec<Self>) -> (SerializerContext, Option<u64>) {
+    pub(crate) fn normalize(builders: Vec<Self>) -> NormalizedStructuralPlan {
         assert!(!builders.is_empty());
-        if builders.iter().all(|b| b.is_empty()) {
-            // No repetition, all-valid
-            let def_meaning = builders
-                .first()
-                .unwrap()
-                .repdefs
-                .iter()
-                .map(|_| DefinitionInterpretation::AllValidItem)
-                .collect::<Vec<_>>();
-            return (
-                SerializerContext {
-                    def_meaning,
-                    rep_levels: LevelBuffer::default(),
-                    spare_rep: LevelBuffer::default(),
-                    def_levels: LevelBuffer::default(),
-                    spare_def: LevelBuffer::default(),
-                    current_rep: 0,
-                    current_def: 0,
-                    current_len: 0,
-                    current_num_specials: 0,
-                    has_fsl: false,
-                },
-                None,
-            );
-        }
-
         let num_layers = builders[0].num_layers();
-        let combined_layers = (0..num_layers)
+        debug_assert!(
+            builders
+                .iter()
+                .all(|builder| builder.num_layers() == num_layers)
+        );
+        let layers = (0..num_layers)
             .map(|layer_index| {
                 Self::concat_layers(
                     builders.iter().map(|b| &b.repdefs[layer_index]),
@@ -1464,47 +1567,10 @@ impl RepDefBuilder {
                 )
             })
             .collect::<Vec<_>>();
-        debug_assert!(
-            builders
-                .iter()
-                .all(|b| b.num_layers() == builders[0].num_layers())
-        );
-
-        let total_len = combined_layers.last().unwrap().num_values()
-            + combined_layers
-                .iter()
-                .map(|l| l.num_specials())
-                .sum::<usize>();
-        let max_rep = combined_layers.iter().map(|l| l.max_rep()).sum::<u16>();
-        let max_def = combined_layers.iter().map(|l| l.max_def()).sum::<u16>();
-        let bits_per_rep = if max_rep > 0 {
-            u64::from(u16::BITS - max_rep.leading_zeros())
-        } else {
-            0
-        };
-        let bits_per_def = if max_def > 0 {
-            u64::from(u16::BITS - max_def.leading_zeros())
-        } else {
-            0
-        };
-        let bits_per_level =
-            (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def);
-
-        let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def);
-        for layer in combined_layers.into_iter() {
-            match layer {
-                RawRepDef::Validity(def) => {
-                    context.record_validity(&def);
-                }
-                RawRepDef::Offsets(rep) => {
-                    context.record_offsets(&rep);
-                }
-                RawRepDef::Fsl(fsl) => {
-                    context.record_fsl(&fsl);
-                }
-            }
+        NormalizedStructuralPlan {
+            layers,
+            dense_all_valid: builders.iter().all(Self::is_empty),
         }
-        (context, bits_per_level)
     }
 }
 
@@ -1514,6 +1580,7 @@ impl RepDefBuilder {
 /// This is used during decoding to create the necessary arrow structures
 #[derive(Debug)]
 pub struct RepDefUnraveler {
+    sparse: Option<SparseStructuralUnraveler>,
     rep_levels: Option<LevelBuffer>,
     def_levels: Option<LevelBuffer>,
     // Maps from definition level to the rep level at which that definition level is visible
@@ -1567,6 +1634,7 @@ impl RepDefUnraveler {
             }
         }
         Self {
+            sparse: None,
             rep_levels,
             def_levels,
             current_def_cmp: 0,
@@ -1578,7 +1646,35 @@ impl RepDefUnraveler {
         }
     }
 
+    pub(crate) fn new_sparse(plan: SparseStructuralPlan) -> Self {
+        Self {
+            sparse: Some(SparseStructuralUnraveler::new(plan)),
+            rep_levels: None,
+            def_levels: None,
+            levels_to_rep: Vec::new(),
+            def_meaning: Arc::new([]),
+            current_def_cmp: 0,
+            current_rep_cmp: 0,
+            current_layer: 0,
+            num_items: 0,
+        }
+    }
+
+    fn ensure_exhausted(&self) -> Result<()> {
+        if let Some(sparse) = &self.sparse {
+            sparse.ensure_exhausted()?;
+        }
+        Ok(())
+    }
+
+    fn is_sparse(&self) -> bool {
+        self.sparse.is_some()
+    }
+
     pub fn is_all_valid(&self) -> bool {
+        if let Some(sparse) = &self.sparse {
+            return sparse.is_all_valid();
+        }
         self.def_levels.is_none() || self.def_meaning[self.current_layer].is_all_valid()
     }
 
@@ -1587,15 +1683,19 @@ impl RepDefUnraveler {
     ///
     /// This is not valid to call when the current level is a struct/primitive layer because
     /// in some cases there may be no rep or def information to know this.
-    pub fn max_lists(&self) -> usize {
+    pub fn max_lists(&self) -> Result<usize> {
+        if let Some(sparse) = &self.sparse {
+            return sparse.max_lists();
+        }
         debug_assert!(
             self.def_meaning[self.current_layer] != DefinitionInterpretation::NullableItem
         );
-        self.rep_levels
+        Ok(self
+            .rep_levels
             .as_ref()
             // Worst case every rep item is max_rep and a new list
             .map(|levels| levels.len())
-            .unwrap_or(0)
+            .unwrap_or(0))
     }
 
     /// Unravels a layer of offsets from the unraveler into the given offset width
@@ -1607,6 +1707,9 @@ impl RepDefUnraveler {
         offsets: &mut Vec<T>,
         validity: Option<&mut BooleanBufferBuilder>,
     ) -> Result<()> {
+        if let Some(sparse) = self.sparse.as_mut() {
+            return sparse.unravel_offsets(offsets, validity);
+        }
         let rep_levels = self
             .rep_levels
             .as_mut()
@@ -1757,18 +1860,25 @@ impl RepDefUnraveler {
         }
     }
 
-    pub fn skip_validity(&mut self) {
+    pub fn skip_validity(&mut self) -> Result<()> {
+        if let Some(sparse) = self.sparse.as_mut() {
+            return sparse.skip_validity();
+        }
         debug_assert!(self.is_all_valid());
         self.current_layer += 1;
+        Ok(())
     }
 
     /// Unravels a layer of validity from the definition levels
-    pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) {
+    pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> {
+        if let Some(sparse) = self.sparse.as_mut() {
+            return sparse.unravel_validity(validity);
+        }
         let meaning = self.def_meaning[self.current_layer];
         if meaning == DefinitionInterpretation::AllValidItem || self.def_levels.is_none() {
             self.current_layer += 1;
             validity.append_n(self.num_items as usize, true);
-            return;
+            return Ok(());
         }
 
         self.current_layer += 1;
@@ -1786,9 +1896,13 @@ impl RepDefUnraveler {
         }) {
             validity.append(is_valid);
         }
+        Ok(())
     }
 
-    pub fn decimate(&mut self, dimension: usize) {
+    pub fn decimate(&mut self, dimension: usize) -> Result<()> {
+        if let Some(sparse) = self.sparse.as_mut() {
+            return sparse.decimate(dimension);
+        }
         if self.rep_levels.is_some() {
             // If we need to support this then I think we need to walk through the rep def levels to find
             // the spots at which we keep.  E.g. if we have:
@@ -1804,7 +1918,7 @@ impl RepDefUnraveler {
             todo!("Not yet supported FSL<...List<...>>");
         }
         let Some(def_levels) = self.def_levels.as_mut() else {
-            return;
+            return Ok(());
         };
         let mut read_idx = 0;
         let mut write_idx = 0;
@@ -1816,6 +1930,7 @@ impl RepDefUnraveler {
             read_idx += dimension;
         }
         def_levels.truncate(write_idx);
+        Ok(())
     }
 }
 
@@ -1835,44 +1950,104 @@ impl RepDefUnraveler {
 #[derive(Debug)]
 pub struct CompositeRepDefUnraveler {
     unravelers: Vec<RepDefUnraveler>,
+    comparisons: Vec<Self>,
 }
 
 impl CompositeRepDefUnraveler {
     pub fn new(unravelers: Vec<RepDefUnraveler>) -> Self {
-        Self { unravelers }
+        Self {
+            unravelers,
+            comparisons: Vec::new(),
+        }
+    }
+
+    pub(crate) fn add_compatibility_check(&mut self, other: Self) {
+        self.comparisons.push(other);
+    }
+
+    pub(crate) fn has_sparse(&self) -> bool {
+        self.unravelers.iter().any(RepDefUnraveler::is_sparse)
+            || self.comparisons.iter().any(Self::has_sparse)
+    }
+
+    pub(crate) fn ensure_exhausted(&self) -> Result<()> {
+        for unraveler in &self.unravelers {
+            unraveler.ensure_exhausted()?;
+        }
+        for comparison in &self.comparisons {
+            comparison.ensure_exhausted()?;
+        }
+        Ok(())
+    }
+
+    fn null_buffers_equal(
+        left: &Option<NullBuffer>,
+        right: &Option<NullBuffer>,
+        expected_len: usize,
+    ) -> bool {
+        match (left, right) {
+            (None, None) => true,
+            (Some(left), Some(right)) => {
+                left.len() == expected_len
+                    && right.len() == expected_len
+                    && left.iter().eq(right.iter())
+            }
+            (None, Some(right)) => right.len() == expected_len && right.null_count() == 0,
+            (Some(left), None) => left.len() == expected_len && left.null_count() == 0,
+        }
+    }
+
+    fn decimate(&mut self, dimension: usize) -> Result<()> {
+        for unraveler in &mut self.unravelers {
+            unraveler.decimate(dimension)?;
+        }
+        for comparison in &mut self.comparisons {
+            comparison.decimate(dimension)?;
+        }
+        Ok(())
     }
 
     /// Unravels a layer of validity
     ///
     /// Returns None if there are no null items in this layer
-    pub fn unravel_validity(&mut self, num_values: usize) -> Option<NullBuffer> {
+    pub fn unravel_validity(&mut self, num_values: usize) -> Result<Option<NullBuffer>> {
         let is_all_valid = self
             .unravelers
             .iter()
             .all(|unraveler| unraveler.is_all_valid());
 
-        if is_all_valid {
+        let validity = if is_all_valid {
             for unraveler in self.unravelers.iter_mut() {
-                unraveler.skip_validity();
+                unraveler.skip_validity()?;
             }
             None
         } else {
             let mut validity = BooleanBufferBuilder::new(num_values);
             for unraveler in self.unravelers.iter_mut() {
-                unraveler.unravel_validity(&mut validity);
+                unraveler.unravel_validity(&mut validity)?;
             }
             Some(NullBuffer::new(validity.finish()))
+        };
+        for comparison in &mut self.comparisons {
+            let other = comparison.unravel_validity(num_values)?;
+            if !Self::null_buffers_equal(&validity, &other, num_values) {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Structural sibling fields have incompatible validity metadata for {num_values} values"
+                    )
+                    .into(),
+                ));
+            }
         }
+        Ok(validity)
     }
 
     pub fn unravel_fsl_validity(
         &mut self,
         num_values: usize,
         dimension: usize,
-    ) -> Option<NullBuffer> {
-        for unraveler in self.unravelers.iter_mut() {
-            unraveler.decimate(dimension);
-        }
+    ) -> Result<Option<NullBuffer>> {
+        self.decimate(dimension)?;
         self.unravel_validity(num_values)
     }
 
@@ -1881,10 +2056,16 @@ impl CompositeRepDefUnraveler {
         &mut self,
     ) -> Result<(OffsetBuffer<T>, Option<NullBuffer>)> {
         let mut is_all_valid = true;
-        let mut max_num_lists = 0;
+        let mut max_num_lists: usize = 0;
         for unraveler in self.unravelers.iter() {
             is_all_valid &= unraveler.is_all_valid();
-            max_num_lists += unraveler.max_lists();
+            max_num_lists = max_num_lists
+                .checked_add(unraveler.max_lists()?)
+                .ok_or_else(|| {
+                    Error::invalid_input_source(
+                        "Combined repetition/definition list count exceeds usize::MAX".into(),
+                    )
+                })?;
         }
 
         let mut validity = if is_all_valid {
@@ -1901,10 +2082,28 @@ impl CompositeRepDefUnraveler {
             unraveler.unravel_offsets(&mut offsets, validity.as_mut())?;
         }
 
-        Ok((
-            OffsetBuffer::new(ScalarBuffer::from(offsets)),
-            validity.map(|mut v| NullBuffer::new(v.finish())),
-        ))
+        let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets));
+        let validity = validity.map(|mut v| NullBuffer::new(v.finish()));
+        for comparison in &mut self.comparisons {
+            let (other_offsets, other_validity) = comparison.unravel_offsets::<T>()?;
+            if offsets.as_ref() != other_offsets.as_ref()
+                || !Self::null_buffers_equal(
+                    &validity,
+                    &other_validity,
+                    offsets.len().saturating_sub(1),
+                )
+            {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Structural sibling fields have incompatible list metadata for {} slots",
+                        offsets.len().saturating_sub(1)
+                    )
+                    .into(),
+                ));
+            }
+        }
+
+        Ok((offsets, validity))
     }
 }
 
@@ -2603,6 +2802,10 @@ impl ControlWordParser {
 mod tests {
     use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 
+    use crate::encodings::logical::primitive::sparse::{
+        SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, SparseValidityMeaning,
+        SparseValiditySet,
+    };
     use crate::repdef::{
         CompositeRepDefUnraveler, DefinitionInterpretation, RepDefUnraveler, SerializedRepDefs,
     };
@@ -2621,6 +2824,31 @@ mod tests {
         OffsetBuffer::<i64>::new(ScalarBuffer::from_iter(values.iter().copied()))
     }
 
+    #[test]
+    fn sparse_sibling_validity_mismatch_is_invalid_input() {
+        let sparse = |positions| {
+            RepDefUnraveler::new_sparse(SparseStructuralPlan {
+                layers: vec![SparseStructuralLayerPlan::Validity {
+                    num_slots: 2,
+                    validity: SparseValiditySet {
+                        meaning: SparseValidityMeaning::NullPositions,
+                        positions,
+                    },
+                }],
+                num_items: 2,
+                num_visible_items: 2,
+            })
+        };
+        let mut repdef = CompositeRepDefUnraveler::new(vec![sparse(SparsePositionSet::Empty)]);
+        repdef.add_compatibility_check(CompositeRepDefUnraveler::new(vec![sparse(
+            SparsePositionSet::Explicit(vec![0]),
+        )]));
+
+        let err = repdef.unravel_validity(2).unwrap_err();
+        assert!(matches!(err, lance_core::Error::InvalidInput { .. }));
+        assert!(err.to_string().contains("incompatible validity metadata"));
+    }
+
     #[test]
     fn test_repdef_empty_offsets() {
         // Empty offsets should serialize without panicking.
@@ -2666,7 +2894,7 @@ mod tests {
         // Note: validity doesn't exactly round-trip because repdef normalizes some of the
         // redundant validity values
         assert_eq!(
-            unraveler.unravel_validity(9),
+            unraveler.unravel_validity(9).unwrap(),
             Some(validity(&[
                 true, true, true, false, false, false, true, true, false
             ]))
@@ -2804,14 +3032,14 @@ mod tests {
         )]);
 
         assert_eq!(
-            unraveler.unravel_validity(8),
+            unraveler.unravel_validity(8).unwrap(),
             Some(validity(&[
                 true, false, true, false, false, false, false, false
             ]))
         );
-        assert_eq!(unraveler.unravel_fsl_validity(4, 2), None);
+        assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None);
         assert_eq!(
-            unraveler.unravel_fsl_validity(2, 2),
+            unraveler.unravel_fsl_validity(2, 2).unwrap(),
             Some(validity(&[true, false]))
         );
     }
@@ -2847,10 +3075,10 @@ mod tests {
             8,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(8), None);
-        assert_eq!(unraveler.unravel_fsl_validity(4, 2), None);
+        assert_eq!(unraveler.unravel_validity(8).unwrap(), None);
+        assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None);
         assert_eq!(
-            unraveler.unravel_fsl_validity(2, 2),
+            unraveler.unravel_fsl_validity(2, 2).unwrap(),
             Some(validity(&[true, false]))
         );
     }
@@ -2928,7 +3156,7 @@ mod tests {
             8,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(6), None);
+        assert_eq!(unraveler.unravel_validity(6).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner());
         assert_eq!(val, None);
@@ -2954,7 +3182,7 @@ mod tests {
             9,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(9), None);
+        assert_eq!(unraveler.unravel_validity(9).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 1, 3, 5, 7, 9]).inner());
         assert_eq!(val, None);
@@ -3018,7 +3246,7 @@ mod tests {
             8,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(6), None);
+        assert_eq!(unraveler.unravel_validity(6).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner());
         assert_eq!(val, None);
@@ -3048,7 +3276,7 @@ mod tests {
             8,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(6), None);
+        assert_eq!(unraveler.unravel_validity(6).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner());
         assert_eq!(val, Some(validity(&[true, false, false, true])));
@@ -3078,7 +3306,7 @@ mod tests {
             8,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(6), None);
+        assert_eq!(unraveler.unravel_validity(6).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner());
         assert_eq!(val, Some(validity(&[true, false, true, true])));
@@ -3106,11 +3334,11 @@ mod tests {
         )]);
 
         assert_eq!(
-            unraveler.unravel_validity(4),
+            unraveler.unravel_validity(4).unwrap(),
             Some(validity(&[false, true, false, false]))
         );
         assert_eq!(
-            unraveler.unravel_validity(4),
+            unraveler.unravel_validity(4).unwrap(),
             Some(validity(&[false, true, false, false]))
         );
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
@@ -3139,14 +3367,14 @@ mod tests {
         )]);
 
         assert_eq!(
-            unraveler.unravel_validity(5),
+            unraveler.unravel_validity(5).unwrap(),
             Some(validity(&[false, false, true, true, false]))
         );
         assert_eq!(
-            unraveler.unravel_validity(5),
+            unraveler.unravel_validity(5).unwrap(),
             Some(validity(&[false, false, true, true, true]))
         );
-        assert_eq!(unraveler.unravel_validity(5), None);
+        assert_eq!(unraveler.unravel_validity(5).unwrap(), None);
     }
 
     #[test]
@@ -3188,7 +3416,7 @@ mod tests {
 
         let mut unraveler = CompositeRepDefUnraveler::new(vec![unravel1, unravel2]);
 
-        assert!(unraveler.unravel_validity(9).is_none());
+        assert!(unraveler.unravel_validity(9).unwrap().is_none());
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(
             off.inner(),
@@ -3484,11 +3712,11 @@ mod tests {
             0,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(0), None);
+        assert_eq!(unraveler.unravel_validity(0).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 0, 0, 0]).inner());
         assert_eq!(val, Some(validity(&[false, false, false])));
-        let val = unraveler.unravel_validity(3).unwrap();
+        let val = unraveler.unravel_validity(3).unwrap().unwrap();
         assert_eq!(val.inner(), validity(&[true, false, true]).inner());
     }
 
@@ -3516,7 +3744,7 @@ mod tests {
             1,
         )]);
 
-        assert_eq!(unraveler.unravel_validity(1), None);
+        assert_eq!(unraveler.unravel_validity(1).unwrap(), None);
         let (off, val) = unraveler.unravel_offsets::<i32>().unwrap();
         assert_eq!(off.inner(), offsets_32(&[0, 1, 1]).inner());
         assert_eq!(val, Some(validity(&[true, false])));
@@ -3547,7 +3775,7 @@ mod tests {
         ]);
 
         assert_eq!(
-            unraveler.unravel_validity(8),
+            unraveler.unravel_validity(8).unwrap(),
             Some(validity(&[
                 true, false, true, false, true, true, true, true
             ]))
@@ -3584,7 +3812,7 @@ mod tests {
         ]);
 
         assert_eq!(
-            unraveler.unravel_validity(4),
+            unraveler.unravel_validity(4).unwrap(),
             Some(validity(&[true, false, true, true]))
         );
         assert_eq!(
@@ -3616,7 +3844,7 @@ mod tests {
         ]);
 
         assert_eq!(
-            unraveler.unravel_validity(8),
+            unraveler.unravel_validity(8).unwrap(),
             Some(validity(&[
                 true, false, true, false, true, true, true, true
             ]))
diff --git a/rust/lance-encoding/src/testing.rs b/rust/lance-encoding/src/testing.rs
index 176083d6d64..9825cb5949d 100644
--- a/rust/lance-encoding/src/testing.rs
+++ b/rust/lance-encoding/src/testing.rs
@@ -666,6 +666,11 @@ fn collect_page_encoding(layout: &PageLayout, actual_chain: &mut Vec<String>) ->
                     collect_page_encoding(inner_layout.as_ref(), actual_chain)?
                 }
             }
+            Layout::SparseLayout(sparse) => {
+                if let Some(value_compression) = &sparse.value_compression {
+                    actual_chain.extend(extract_array_encoding_chain(value_compression));
+                }
+            }
         }
     }
 
diff --git a/rust/lance-file/Cargo.toml b/rust/lance-file/Cargo.toml
index f08cd3457aa..56536e840e9 100644
--- a/rust/lance-file/Cargo.toml
+++ b/rust/lance-file/Cargo.toml
@@ -61,5 +61,9 @@ features = ["protoc"]
 name = "reader"
 harness = false
 
+[[bench]]
+name = "schema"
+harness = false
+
 [lints]
 workspace = true
diff --git a/rust/lance-file/benches/schema.rs b/rust/lance-file/benches/schema.rs
new file mode 100644
index 00000000000..b8da23d9777
--- /dev/null
+++ b/rust/lance-file/benches/schema.rs
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::hint::black_box;
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use lance_core::datatypes::Schema;
+use lance_file::{datatypes::Fields, format::pb};
+
+fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field {
+    pb::Field {
+        id,
+        parent_id,
+        name,
+        logical_type: logical_type.to_owned(),
+        ..Default::default()
+    }
+}
+
+/// Builds a pre-order flat schema with `num_physical_columns` physical leaves.
+///
+/// Each struct contributes one parent and two `int32` leaves. Root fields use
+/// `-1` as `parent_id`, and each struct consumes a three-ID block.
+fn wide_two_leaf_structs(num_physical_columns: usize) -> Fields {
+    assert_eq!(num_physical_columns % 2, 0);
+    let num_structs = num_physical_columns / 2;
+    let mut fields = Vec::with_capacity(num_structs + num_physical_columns);
+
+    for struct_index in 0..num_structs {
+        let parent_id = (struct_index * 3) as i32;
+        fields.push(proto_field(
+            parent_id,
+            -1,
+            format!("struct_{struct_index}"),
+            "struct",
+        ));
+        fields.push(proto_field(
+            parent_id + 1,
+            parent_id,
+            format!("left_{struct_index}"),
+            "int32",
+        ));
+        fields.push(proto_field(
+            parent_id + 2,
+            parent_id,
+            format!("right_{struct_index}"),
+            "int32",
+        ));
+    }
+
+    Fields(fields)
+}
+
+fn bench_schema_reconstruction(c: &mut Criterion) {
+    let mut group = c.benchmark_group("schema_from_flat_fields");
+
+    for num_physical_columns in [1024, 4096, 16_384, 65_536] {
+        let fields = wide_two_leaf_structs(num_physical_columns);
+        group.throughput(Throughput::Elements(fields.0.len() as u64));
+        group.bench_with_input(
+            BenchmarkId::new("physical_columns", num_physical_columns),
+            &fields,
+            |bencher, fields| {
+                bencher.iter(|| Schema::try_from(black_box(fields)).unwrap());
+            },
+        );
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_schema_reconstruction);
+criterion_main!(benches);
diff --git a/rust/lance-file/src/datatypes.rs b/rust/lance-file/src/datatypes.rs
index ac6a8d7b293..c31cb5c97e7 100644
--- a/rust/lance-file/src/datatypes.rs
+++ b/rust/lance-file/src/datatypes.rs
@@ -99,6 +99,32 @@ impl From<&Field> for pb::Field {
 
 pub struct Fields(pub Vec<pb::Field>);
 
+struct FieldNode {
+    field: Field,
+    child_indices: Vec<usize>,
+}
+
+/// Searches in pre-order depth-first order and returns the first matching node,
+/// preserving the legacy parent tie-break for duplicate field IDs.
+fn first_field_index_by_id(
+    nodes: &[FieldNode],
+    root_indices: &[usize],
+    field_id: i32,
+) -> Option<usize> {
+    let mut to_visit = Vec::with_capacity(nodes.len());
+    to_visit.extend(root_indices.iter().rev().copied());
+
+    while let Some(node_index) = to_visit.pop() {
+        let node = &nodes[node_index];
+        if node.field.id == field_id {
+            return Some(node_index);
+        }
+        to_visit.extend(node.child_indices.iter().rev().copied());
+    }
+
+    None
+}
+
 impl From<&Field> for Fields {
     fn from(field: &Field) -> Self {
         let mut protos = vec![pb::Field::from(field)];
@@ -107,24 +133,119 @@ impl From<&Field> for Fields {
     }
 }
 
-/// Convert list of protobuf `Field` to a Schema.
-impl From<&Fields> for Schema {
-    fn from(fields: &Fields) -> Self {
-        let mut schema = Self {
-            fields: vec![],
-            metadata: HashMap::default(),
-        };
-
-        fields.0.iter().for_each(|f| {
-            if f.parent_id == -1 {
-                schema.fields.push(Field::from(f));
+/// Reconstruct a schema from a flat, pre-order protobuf field list.
+///
+/// Parent fields must appear before their children. Historical manifests may
+/// contain duplicate field IDs, so an ID may not identify a unique parent. For
+/// those references, reconstruction preserves the legacy
+/// [`Schema::mut_field_by_id`] tie-break by selecting the first matching field
+/// in pre-order depth-first traversal.
+///
+/// # Examples
+///
+/// ```
+/// use lance_core::datatypes::Schema;
+/// use lance_file::{datatypes::Fields, format::pb};
+///
+/// let field = pb::Field {
+///     id: 0,
+///     parent_id: -1,
+///     name: "value".to_owned(),
+///     logical_type: "int32".to_owned(),
+///     ..Default::default()
+/// };
+/// let fields = Fields(vec![field]);
+/// let schema = Schema::try_from(&fields)?;
+/// assert_eq!(schema.fields[0].name, "value");
+/// # Ok::<(), lance_core::Error>(())
+/// ```
+impl TryFrom<&Fields> for Schema {
+    type Error = Error;
+
+    fn try_from(fields: &Fields) -> Result<Self> {
+        let mut nodes: Vec<FieldNode> = Vec::with_capacity(fields.0.len());
+        let mut root_indices = Vec::with_capacity(fields.0.len());
+        let mut field_indices: HashMap<i32, Option<usize>> = HashMap::with_capacity(fields.0.len());
+
+        for proto_field in &fields.0 {
+            let parent_index = if proto_field.parent_id == -1 {
+                None
             } else {
-                let parent = schema.mut_field_by_id(f.parent_id).unwrap();
-                parent.children.push(Field::from(f));
+                let parent_index = match field_indices.get(&proto_field.parent_id) {
+                    Some(Some(parent_index)) => *parent_index,
+                    Some(None) => {
+                        // Duplicate IDs are invalid but occur in historical
+                        // manifests. Match the legacy tree traversal only for
+                        // these ambiguous parent references so valid schemas
+                        // retain the linear fast path.
+                        first_field_index_by_id(&nodes, &root_indices, proto_field.parent_id)
+                            .ok_or_else(|| {
+                                Error::internal(format!(
+                                    "Duplicate field id {} has no existing arena node",
+                                    proto_field.parent_id
+                                ))
+                            })?
+                    }
+                    None => {
+                        return Err(Error::schema(format!(
+                            "Field '{}' (id={}) references parent id {}, which must appear earlier in the protobuf field list",
+                            proto_field.name, proto_field.id, proto_field.parent_id
+                        )));
+                    }
+                };
+                Some(parent_index)
+            };
+
+            let node_index = nodes.len();
+            if let Some(parent_index) = parent_index {
+                nodes[parent_index].child_indices.push(node_index);
+            } else {
+                root_indices.push(node_index);
             }
-        });
+            nodes.push(FieldNode {
+                field: Field::from(proto_field),
+                child_indices: Vec::new(),
+            });
+
+            field_indices
+                .entry(proto_field.id)
+                .and_modify(|field_index| *field_index = None)
+                .or_insert(Some(node_index));
+        }
 
-        schema
+        let mut fields_by_node = Vec::with_capacity(nodes.len());
+        fields_by_node.resize_with(nodes.len(), || None);
+        for (node_index, mut node) in nodes.into_iter().enumerate().rev() {
+            node.field.children.reserve(node.child_indices.len());
+            for child_index in node.child_indices {
+                let child = fields_by_node
+                    .get_mut(child_index)
+                    .and_then(Option::take)
+                    .ok_or_else(|| {
+                        Error::internal(format!(
+                            "Schema field arena node {child_index} was not materialized before its parent"
+                        ))
+                    })?;
+                node.field.children.push(child);
+            }
+            fields_by_node[node_index] = Some(node.field);
+        }
+
+        let fields = root_indices
+            .into_iter()
+            .map(|root_index| {
+                fields_by_node[root_index].take().ok_or_else(|| {
+                    Error::internal(format!(
+                        "Schema field arena root node {root_index} was not materialized"
+                    ))
+                })
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Self {
+            fields,
+            metadata: HashMap::default(),
+        })
     }
 }
 
@@ -133,9 +254,28 @@ pub struct FieldsWithMeta {
     pub metadata: HashMap<String, Vec<u8>>,
 }
 
-/// Convert list of protobuf `Field` and Metadata to a Schema.
-impl From<FieldsWithMeta> for Schema {
-    fn from(fields_with_meta: FieldsWithMeta) -> Self {
+/// Reconstruct a schema from flat protobuf fields and schema metadata.
+///
+/// # Examples
+///
+/// ```
+/// use std::collections::HashMap;
+///
+/// use lance_core::datatypes::Schema;
+/// use lance_file::datatypes::{Fields, FieldsWithMeta};
+///
+/// let fields = FieldsWithMeta {
+///     fields: Fields(Vec::new()),
+///     metadata: HashMap::from([("owner".to_owned(), b"lance".to_vec())]),
+/// };
+/// let schema = Schema::try_from(fields)?;
+/// assert_eq!(schema.metadata["owner"], "lance");
+/// # Ok::<(), lance_core::Error>(())
+/// ```
+impl TryFrom<FieldsWithMeta> for Schema {
+    type Error = Error;
+
+    fn try_from(fields_with_meta: FieldsWithMeta) -> Result<Self> {
         let lance_metadata = fields_with_meta
             .metadata
             .into_iter()
@@ -145,11 +285,11 @@ impl From<FieldsWithMeta> for Schema {
             })
             .collect();
 
-        let schema_with_fields = Self::from(&fields_with_meta.fields);
-        Self {
+        let schema_with_fields = Self::try_from(&fields_with_meta.fields)?;
+        Ok(Self {
             fields: schema_with_fields.fields,
             metadata: lance_metadata,
-        }
+        })
     }
 }
 
@@ -270,14 +410,27 @@ pub async fn populate_schema_dictionary(schema: &mut Schema, reader: &dyn Reader
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use arrow_schema::DataType;
     use arrow_schema::Field as ArrowField;
     use arrow_schema::Fields as ArrowFields;
     use arrow_schema::Schema as ArrowSchema;
+    use lance_core::Error;
     use lance_core::datatypes::Schema;
-    use std::collections::HashMap;
 
     use super::{Fields, FieldsWithMeta};
+    use crate::format::pb;
+
+    fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field {
+        pb::Field {
+            id,
+            parent_id,
+            name,
+            logical_type: logical_type.to_owned(),
+            ..Default::default()
+        }
+    }
 
     #[test]
     fn test_schema_set_ids() {
@@ -317,10 +470,120 @@ mod tests {
         let expected_schema = Schema::try_from(&arrow_schema).unwrap();
         let fields_with_meta: FieldsWithMeta = (&expected_schema).into();
 
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta).unwrap();
         assert_eq!(expected_schema, schema);
     }
 
+    #[test]
+    fn test_reconstruct_wide_nested_schema() {
+        const NUM_STRUCTS: usize = 4096;
+
+        let mut proto_fields = Vec::with_capacity(NUM_STRUCTS * 3);
+        for struct_index in 0..NUM_STRUCTS {
+            let parent_id = (struct_index * 3) as i32;
+            proto_fields.push(proto_field(
+                parent_id,
+                -1,
+                format!("struct_{struct_index}"),
+                "struct",
+            ));
+            proto_fields.push(proto_field(
+                parent_id + 1,
+                parent_id,
+                format!("left_{struct_index}"),
+                "int32",
+            ));
+            proto_fields.push(proto_field(
+                parent_id + 2,
+                parent_id,
+                format!("right_{struct_index}"),
+                "int32",
+            ));
+        }
+
+        let fields = Fields(proto_fields);
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), NUM_STRUCTS);
+        for (struct_index, field) in schema.fields.iter().enumerate() {
+            let parent_id = (struct_index * 3) as i32;
+            assert_eq!(field.id, parent_id);
+            assert_eq!(field.name, format!("struct_{struct_index}"));
+            assert_eq!(field.children.len(), 2);
+            assert_eq!(field.children[0].id, parent_id + 1);
+            assert_eq!(field.children[0].name, format!("left_{struct_index}"));
+            assert_eq!(field.children[1].id, parent_id + 2);
+            assert_eq!(field.children[1].name, format!("right_{struct_index}"));
+        }
+    }
+
+    #[test]
+    fn test_reconstruct_deep_nested_schema() {
+        const DEPTH: usize = 1024;
+
+        let proto_fields = (0..DEPTH)
+            .map(|depth| {
+                proto_field(
+                    depth as i32,
+                    if depth == 0 { -1 } else { depth as i32 - 1 },
+                    format!("level_{depth}"),
+                    if depth + 1 == DEPTH {
+                        "int32"
+                    } else {
+                        "struct"
+                    },
+                )
+            })
+            .collect();
+
+        let fields = Fields(proto_fields);
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), 1);
+        let mut field = &schema.fields[0];
+        for depth in 0..DEPTH {
+            assert_eq!(field.id, depth as i32);
+            assert_eq!(field.name, format!("level_{depth}"));
+            if depth + 1 == DEPTH {
+                assert!(field.children.is_empty());
+            } else {
+                assert_eq!(field.children.len(), 1);
+                field = &field.children[0];
+            }
+        }
+    }
+
+    #[test]
+    fn test_reconstruct_schema_reports_missing_parent() {
+        let fields = Fields(vec![proto_field(7, 42, "child".to_owned(), "int32")]);
+
+        let error = Schema::try_from(&fields).unwrap_err();
+        assert!(matches!(&error, Error::Schema { .. }));
+        assert!(
+            error.to_string().contains(
+                "Field 'child' (id=7) references parent id 42, which must appear earlier"
+            )
+        );
+    }
+
+    #[test]
+    fn test_reconstruct_schema_preserves_legacy_duplicate_id_match() {
+        let fields = Fields(vec![
+            proto_field(1, -1, "root_a".to_owned(), "struct"),
+            proto_field(2, -1, "root_b".to_owned(), "struct"),
+            proto_field(2, 1, "nested_duplicate".to_owned(), "struct"),
+            proto_field(3, 2, "child".to_owned(), "int32"),
+        ]);
+
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), 2);
+        assert_eq!(schema.fields[0].name, "root_a");
+        assert_eq!(schema.fields[0].children.len(), 1);
+        assert_eq!(schema.fields[0].children[0].name, "nested_duplicate");
+        assert_eq!(schema.fields[0].children[0].children.len(), 1);
+        assert_eq!(schema.fields[0].children[0].children[0].name, "child");
+        assert_eq!(schema.fields[1].name, "root_b");
+        assert!(schema.fields[1].children.is_empty());
+    }
+
     #[test]
     fn test_clustering_key_roundtrip() {
         let arrow_schema = ArrowSchema::new(vec![
@@ -351,7 +614,7 @@ mod tests {
 
         // Round-trip through protobuf
         let fields_with_meta: FieldsWithMeta = (&schema).into();
-        let restored = Schema::from(fields_with_meta);
+        let restored = Schema::try_from(fields_with_meta).unwrap();
 
         let ck2 = restored.unenforced_clustering_key();
         assert_eq!(ck2.len(), 2);
diff --git a/rust/lance-file/src/io.rs b/rust/lance-file/src/io.rs
index 1a8edf92b08..86e5189a81d 100644
--- a/rust/lance-file/src/io.rs
+++ b/rust/lance-file/src/io.rs
@@ -55,14 +55,27 @@ impl EncodingsIo for LanceEncodingsIo {
     ) -> BoxFuture<'static, lance_core::Result<Vec<bytes::Bytes>>> {
         let mut split_ranges = Vec::new();
         let mut split_indices = Vec::new(); // Track which original range each split came from
+        // Large ranges (above read_chunk_size) will be split into
+        // multiple reads.  Empty ranges will skip the I/O layer
+        // entirely.  If we have either of these we will need to
+        // reassemble our results, inserting empties and merging parts
+        let mut needs_reassembly = false;
 
         // Split large ranges into smaller chunks
         //
         // TODO: consider read_chunk_size before submitting requests.
         for (idx, range) in ranges.iter().enumerate() {
+            if range.start == range.end {
+                // EncodingsIo requires one result per input range. Zero-length
+                // ranges schedule no I/O, so their empty results are restored
+                // after the non-empty requests complete.
+                needs_reassembly = true;
+                continue;
+            }
             let range_size = range.end - range.start;
 
             if range_size > self.read_chunk_size {
+                needs_reassembly = true;
                 let num_chunks = range_size.div_ceil(self.read_chunk_size);
                 let chunk_size = range_size / num_chunks;
 
@@ -87,34 +100,48 @@ impl EncodingsIo for LanceEncodingsIo {
         async move {
             let split_results = fut.await?;
 
-            // Fast path: if no splitting occurred, return results directly
-            if split_results.len() == ranges.len() {
+            if split_results.len() != split_indices.len() {
+                return Err(lance_core::Error::internal(format!(
+                    "Encoding I/O returned {} results for {} requested range chunks",
+                    split_results.len(),
+                    split_indices.len()
+                )));
+            }
+            if !needs_reassembly {
                 return Ok(split_results);
             }
 
-            // Slow path: reassemble split results
             let mut results = vec![Vec::new(); ranges.len()];
 
-            for (split_result, &orig_idx) in split_results.iter().zip(split_indices.iter()) {
-                results[orig_idx].push(split_result.clone());
+            for (split_result, orig_idx) in split_results.into_iter().zip(split_indices) {
+                results[orig_idx].push(split_result);
             }
 
-            Ok(results
-                .into_iter()
-                .map(|chunks| {
-                    if chunks.len() == 1 {
-                        chunks.into_iter().next().unwrap()
-                    } else {
-                        // Concatenate multiple chunks
-                        let total_size: usize = chunks.iter().map(|c| c.len()).sum();
-                        let mut combined = Vec::with_capacity(total_size);
-                        for chunk in chunks {
-                            combined.extend_from_slice(&chunk);
-                        }
-                        bytes::Bytes::from(combined)
+            let mut reassembled = Vec::with_capacity(ranges.len());
+            for (range, chunks) in ranges.iter().zip(results) {
+                if chunks.is_empty() {
+                    if range.start == range.end {
+                        reassembled.push(bytes::Bytes::new());
+                        continue;
                     }
-                })
-                .collect())
+                    return Err(lance_core::Error::internal(format!(
+                        "Encoding I/O returned no data for non-empty range {}..{}",
+                        range.start, range.end
+                    )));
+                }
+                if chunks.len() == 1 {
+                    reassembled.push(chunks[0].clone());
+                    continue;
+                }
+
+                let total_size: usize = chunks.iter().map(|c| c.len()).sum();
+                let mut combined = Vec::with_capacity(total_size);
+                for chunk in chunks {
+                    combined.extend_from_slice(&chunk);
+                }
+                reassembled.push(bytes::Bytes::from(combined));
+            }
+            Ok(reassembled)
         }
         .boxed()
     }
diff --git a/rust/lance-file/src/previous/format/metadata.rs b/rust/lance-file/src/previous/format/metadata.rs
index 11ba00c3243..209d0733cec 100644
--- a/rust/lance-file/src/previous/format/metadata.rs
+++ b/rust/lance-file/src/previous/format/metadata.rs
@@ -62,10 +62,10 @@ impl TryFrom<pb::Metadata> for Metadata {
             manifest_position: Some(m.manifest_position as usize),
             stats_metadata: if let Some(stats_meta) = m.statistics {
                 Some(StatisticsMetadata {
-                    schema: Schema::from(FieldsWithMeta {
+                    schema: Schema::try_from(FieldsWithMeta {
                         fields: Fields(stats_meta.schema),
                         metadata: Default::default(),
-                    }),
+                    })?,
                     leaf_field_ids: stats_meta.fields,
                     page_table_position: stats_meta.page_table_position as usize,
                 })
diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs
index 048cf550d66..a4108020859 100644
--- a/rust/lance-file/src/reader.rs
+++ b/rust/lance-file/src/reader.rs
@@ -536,6 +536,30 @@ fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) {
     (contributes, recurse)
 }
 
+// Count the V2.1 physical columns required to reconstruct a projected field.
+// This is the same DFS shape consumed by `ColumnInfoIter`: ordinary structural
+// nodes are transparent and leaves contribute columns. Indexed metadata loading
+// can therefore compact any ordinary structural projection into 0..N while
+// preserving this order.
+//
+// Blob and packed-struct fields remain unsupported by indexed projection. Their
+// opaque decode semantics are handled by the existing full-metadata reader.
+fn indexed_projection_column_count(field: &Field) -> Option<usize> {
+    if field.is_blob() || field.is_packed_struct() {
+        return None;
+    }
+
+    let (contributes, recurse) = field_column_shape(field, true);
+    let initial = usize::from(contributes);
+    if !recurse {
+        return Some(initial);
+    }
+
+    field.children.iter().try_fold(initial, |count, child| {
+        count.checked_add(indexed_projection_column_count(child)?)
+    })
+}
+
 // Whether a field's children each cover the same rows as the field itself. Struct
 // children do (one value per parent row), so they must share its length. List,
 // map, and fixed-size-list items have an independent cardinality (item count, not
@@ -919,11 +943,17 @@ impl FileReader {
         let mut global_bufs_cursor = Cursor::new(gbo_bytes);
 
         let mut global_buffers = Vec::with_capacity(footer.num_global_buffers as usize);
-        for _ in 0..footer.num_global_buffers {
+        for buffer_index in 0..footer.num_global_buffers {
             let buf_pos = global_bufs_cursor.read_u64::<LittleEndian>()?;
-            assert!(
-                version < LanceFileVersion::V2_1 || buf_pos % PAGE_BUFFER_ALIGNMENT as u64 == 0
-            );
+            if version >= LanceFileVersion::V2_1 && buf_pos % PAGE_BUFFER_ALIGNMENT as u64 != 0 {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "Global buffer {} position {} is not aligned to {} bytes",
+                        buffer_index, buf_pos, PAGE_BUFFER_ALIGNMENT
+                    )
+                    .into(),
+                ));
+            }
             let buf_size = global_bufs_cursor.read_u64::<LittleEndian>()?;
             global_buffers.push(BufferDescriptor {
                 position: buf_pos,
@@ -961,7 +991,7 @@ impl FileReader {
             fields: Fields(pb_schema.fields),
             metadata: pb_schema.metadata,
         };
-        let schema = lance_core::datatypes::Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)?;
         Ok((num_rows, schema))
     }
 
@@ -1021,7 +1051,7 @@ impl FileReader {
         let num_data_bytes = footer.column_meta_start - num_global_buffer_bytes;
         let num_column_metadata_bytes = footer.global_buff_offsets_start - footer.column_meta_start;
 
-        let column_infos = Self::meta_to_col_infos(column_metadatas.as_slice(), file_version);
+        let column_infos = Self::meta_to_col_infos(column_metadatas.as_slice(), file_version)?;
 
         // The tail read above already pulled in any global buffer that lives within
         // the captured window. Copy those user buffers (index >= 1; the schema at 0
@@ -1129,74 +1159,139 @@ impl FileReader {
         Self::read_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))).await
     }
 
-    fn fetch_encoding<M: Default + Name + Sized>(encoding: &pbfile::Encoding) -> M {
+    fn fetch_encoding<M: Default + Name + Sized>(encoding: &pbfile::Encoding) -> Result<M> {
         match &encoding.location {
-            Some(pbfile::encoding::Location::Indirect(_)) => todo!(),
+            Some(pbfile::encoding::Location::Indirect(_)) => Err(Error::invalid_input_source(
+                "Indirect file encodings are not supported".into(),
+            )),
             Some(pbfile::encoding::Location::Direct(encoding)) => {
                 let encoding_buf = Bytes::from(encoding.encoding.clone());
-                let encoding_any = prost_types::Any::decode(encoding_buf).unwrap();
-                encoding_any.to_msg::<M>().unwrap()
+                let encoding_any = prost_types::Any::decode(encoding_buf).map_err(|error| {
+                    Error::invalid_input_source(
+                        format!("Invalid direct {} encoding envelope: {error}", M::NAME).into(),
+                    )
+                })?;
+                encoding_any.to_msg::<M>().map_err(|error| {
+                    Error::invalid_input_source(
+                        format!("Invalid direct {} encoding: {error}", M::NAME).into(),
+                    )
+                })
             }
-            Some(pbfile::encoding::Location::None(_)) => panic!(),
-            None => panic!(),
+            Some(pbfile::encoding::Location::None(_)) => Err(Error::invalid_input_source(
+                format!("Missing {} encoding description", M::NAME).into(),
+            )),
+            None => Err(Error::invalid_input_source(
+                format!("Missing {} encoding location", M::NAME).into(),
+            )),
         }
     }
 
     fn meta_to_col_infos(
         column_metadatas: &[pbfile::ColumnMetadata],
         file_version: LanceFileVersion,
-    ) -> Vec<Arc<ColumnInfo>> {
+    ) -> Result<Vec<Arc<ColumnInfo>>> {
         column_metadatas
             .iter()
             .enumerate()
             .map(|(col_idx, col_meta)| {
-                Self::meta_to_col_info(col_idx as u32, col_meta, file_version)
+                let col_idx = u32::try_from(col_idx).map_err(|_| {
+                    Error::invalid_input_source("File has more than u32::MAX columns".into())
+                })?;
+                Self::meta_to_col_info(col_idx, col_meta, file_version)
             })
-            .collect::<Vec<_>>()
+            .collect()
     }
 
     fn meta_to_col_info(
         col_idx: u32,
         col_meta: &pbfile::ColumnMetadata,
         file_version: LanceFileVersion,
-    ) -> Arc<ColumnInfo> {
+    ) -> Result<Arc<ColumnInfo>> {
         let page_infos = col_meta
             .pages
             .iter()
-            .map(|page| {
+            .enumerate()
+            .map(|(page_idx, page)| {
                 let num_rows = page.length;
                 let encoding = match file_version {
                     LanceFileVersion::V2_0 => {
                         PageEncoding::Legacy(Self::fetch_encoding::<pbenc::ArrayEncoding>(
-                            page.encoding.as_ref().unwrap(),
-                        ))
+                            page.encoding.as_ref().ok_or_else(|| {
+                                Error::invalid_input_source(
+                                    format!(
+                                        "Column {} page {} is missing its encoding",
+                                        col_idx, page_idx
+                                    )
+                                    .into(),
+                                )
+                            })?,
+                        )?)
+                    }
+                    _ => {
+                        PageEncoding::Structural(Self::fetch_encoding::<pbenc21::PageLayout>(
+                            page.encoding.as_ref().ok_or_else(|| {
+                                Error::invalid_input_source(
+                                    format!(
+                                        "Column {} page {} is missing its encoding",
+                                        col_idx, page_idx
+                                    )
+                                    .into(),
+                                )
+                            })?,
+                        )?)
                     }
-                    _ => PageEncoding::Structural(Self::fetch_encoding::<pbenc21::PageLayout>(
-                        page.encoding.as_ref().unwrap(),
-                    )),
                 };
+                if page.buffer_offsets.len() != page.buffer_sizes.len() {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "Column {} page {} has {} buffer offsets but {} buffer sizes",
+                            col_idx,
+                            page_idx,
+                            page.buffer_offsets.len(),
+                            page.buffer_sizes.len()
+                        )
+                        .into(),
+                    ));
+                }
                 let buffer_offsets_and_sizes = Arc::from(
                     page.buffer_offsets
                         .iter()
                         .zip(page.buffer_sizes.iter())
-                        .map(|(offset, size)| {
-                            // Starting with version 2.1 we can assert that page buffers are aligned
-                            assert!(
-                                file_version < LanceFileVersion::V2_1
-                                    || offset % PAGE_BUFFER_ALIGNMENT as u64 == 0
-                            );
-                            (*offset, *size)
+                        .map(|(offset, size)| -> Result<_> {
+                            if file_version >= LanceFileVersion::V2_1
+                                && offset % PAGE_BUFFER_ALIGNMENT as u64 != 0
+                            {
+                                return Err(Error::invalid_input_source(
+                                    format!(
+                                        "Column {} page {} buffer offset {} is not aligned to {} bytes",
+                                        col_idx, page_idx, offset, PAGE_BUFFER_ALIGNMENT
+                                    )
+                                    .into(),
+                                ));
+                            }
+                            Ok((*offset, *size))
                         })
-                        .collect::<Vec<_>>(),
+                        .collect::<Result<Vec<_>>>()?,
                 );
-                PageInfo {
+                Ok(PageInfo {
                     buffer_offsets_and_sizes,
                     encoding,
                     num_rows,
                     priority: page.priority,
-                }
+                })
             })
-            .collect::<Vec<_>>();
+            .collect::<Result<Vec<_>>>()?;
+        if col_meta.buffer_offsets.len() != col_meta.buffer_sizes.len() {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "Column {} has {} buffer offsets but {} buffer sizes",
+                    col_idx,
+                    col_meta.buffer_offsets.len(),
+                    col_meta.buffer_sizes.len()
+                )
+                .into(),
+            ));
+        }
         let buffer_offsets_and_sizes = Arc::from(
             col_meta
                 .buffer_offsets
@@ -1205,12 +1300,16 @@ impl FileReader {
                 .map(|(offset, size)| (*offset, *size))
                 .collect::<Vec<_>>(),
         );
-        Arc::new(ColumnInfo {
+        Ok(Arc::new(ColumnInfo {
             index: col_idx,
             page_infos: Arc::from(page_infos),
             buffer_offsets_and_sizes,
-            encoding: Self::fetch_encoding(col_meta.encoding.as_ref().unwrap()),
-        })
+            encoding: Self::fetch_encoding(col_meta.encoding.as_ref().ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!("Column {} is missing its encoding", col_idx).into(),
+                )
+            })?)?,
+        }))
     }
 
     fn validate_projection(
@@ -1836,12 +1935,18 @@ impl FileMetadataProvider {
         projection: &ReaderProjection,
         version: LanceFileVersion,
     ) -> bool {
-        version >= LanceFileVersion::V2_1
-            && !projection.schema.fields.is_empty()
-            && projection.schema.fields.len() == projection.column_indices.len()
-            && projection.schema.fields.iter().all(|field| {
-                field.children.is_empty() && !field.is_blob() && !field.is_packed_struct()
+        if version < LanceFileVersion::V2_1 || projection.schema.fields.is_empty() {
+            return false;
+        }
+
+        projection
+            .schema
+            .fields
+            .iter()
+            .try_fold(0usize, |count, field| {
+                count.checked_add(indexed_projection_column_count(field)?)
             })
+            == Some(projection.column_indices.len())
     }
 
     fn validate_indexed_projection(
@@ -1871,7 +1976,7 @@ impl FileMetadataProvider {
         }
         if !Self::supports_indexed_projection(projection, metadata_index.version) {
             return Err(Error::not_supported(format!(
-                "lazy column metadata loading only supports direct V2.1+ top-level physical column projections; got file version {:?}, {} schema fields, and {} column indices",
+                "lazy column metadata loading requires a V2.1+ ordinary structural projection without blob or packed-struct fields whose physical-column count matches the projection; got file version {:?}, {} schema fields, and {} column indices",
                 metadata_index.version,
                 projection.schema.fields.len(),
                 projection.column_indices.len()
@@ -1939,14 +2044,14 @@ impl FileMetadataProvider {
                 .collect::<Vec<_>>();
             let metadata_bytes = io.submit_request(ranges, 0).await?;
             for ((result_index, column_index, _), bytes) in
-                missing_columns.into_iter().zip(metadata_bytes.into_iter())
+                missing_columns.into_iter().zip(metadata_bytes)
             {
                 let column_metadata = pbfile::ColumnMetadata::decode(bytes)?;
                 let column_info = FileReader::meta_to_col_info(
                     column_index,
                     &column_metadata,
                     metadata_index.version,
-                );
+                )?;
                 let cached = Arc::new(CachedColumnMetadata {
                     column_metadata,
                     column_info: column_info.clone(),
@@ -2108,7 +2213,32 @@ impl FileReadCore {
                     column_infos.len()
                 ))
             })?;
-            Ok(info.page_infos.iter().map(|page| page.num_rows).sum())
+            info.page_infos.iter().try_fold(0_u64, |rows, page| {
+                let page_rows = match &page.encoding {
+                    PageEncoding::Structural(layout) => match &layout.layout {
+                        Some(pbenc21::page_layout::Layout::SparseLayout(sparse)) => sparse
+                            .structural_layers
+                            .first()
+                            .and_then(|layer| layer.layer.as_ref())
+                            .map_or(page.num_rows, |layer| match layer {
+                                pbenc21::sparse_structural_layer::Layer::Validity(layer) => {
+                                    layer.num_slots
+                                }
+                                pbenc21::sparse_structural_layer::Layer::List(layer) => {
+                                    layer.num_slots
+                                }
+                                pbenc21::sparse_structural_layer::Layer::FixedSizeList(layer) => {
+                                    layer.num_slots
+                                }
+                            }),
+                        _ => page.num_rows,
+                    },
+                    _ => page.num_rows,
+                };
+                rows.checked_add(page_rows).ok_or_else(|| {
+                    Error::invalid_input_source("Column row count overflows u64".into())
+                })
+            })
         };
         let column_indices = &prepared.decoder_projection.column_indices;
         let fields = &prepared.decoder_projection.schema.fields;
@@ -2512,7 +2642,7 @@ impl EncodedBatchReaderExt for EncodedBatch {
             footer.minor_version as u32,
         )?;
 
-        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version);
+        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version)?;
 
         Ok(Self {
             data: bytes,
@@ -2561,7 +2691,7 @@ impl EncodedBatchReaderExt for EncodedBatch {
         let column_metadatas =
             FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?;
 
-        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version);
+        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version)?;
 
         Ok(Self {
             data: bytes,
@@ -2585,20 +2715,24 @@ mod tests {
     };
 
     use arrow_array::{
-        RecordBatch, UInt32Array,
+        Int32Array, ListArray, RecordBatch, RecordBatchIterator, UInt32Array,
         types::{Float64Type, Int32Type},
     };
+    use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
     use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema};
     use bytes::Bytes;
     use futures::{StreamExt, prelude::stream::TryStreamExt};
     use lance_arrow::{BLOB_META_KEY, RecordBatchExt};
     use lance_core::{ArrowResult, datatypes::Schema};
-    use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch};
+    use lance_datagen::{ArrayGeneratorExt, BatchCount, ByteCount, RowCount, array, gen_batch};
     use lance_encoding::{
+        constants::{STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_SPARSE},
         decoder::{
-            DecodeBatchScheduler, DecoderPlugins, FilterExpression, ReadBatchTask, decode_batch,
+            DecodeBatchScheduler, DecoderPlugins, FilterExpression, PageEncoding, ReadBatchTask,
+            decode_batch,
         },
         encoder::{EncodedBatch, EncodingOptions, default_encoding_strategy, encode_batch},
+        format::pb21,
         version::LanceFileVersion,
     };
     use lance_io::{stream::RecordBatchStream, utils::CachedFileSize};
@@ -2614,6 +2748,135 @@ mod tests {
     use crate::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions};
     use lance_encoding::decoder::DecoderConfig;
 
+    #[tokio::test]
+    async fn sparse_file_writer_reader_scan_range_and_take_roundtrip() {
+        let fs = FsFixture::default();
+        let sparse_metadata = HashMap::from([(
+            STRUCTURAL_ENCODING_META_KEY.to_string(),
+            STRUCTURAL_ENCODING_SPARSE.to_string(),
+        )]);
+        let value_field =
+            Field::new("values", DataType::Int32, true).with_metadata(sparse_metadata.clone());
+        let item_field = Arc::new(Field::new("item", DataType::Int32, true));
+        let list_field = Field::new("items", DataType::List(item_field.clone()), true)
+            .with_metadata(sparse_metadata);
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![value_field, list_field]));
+        let list = ListArray::try_new(
+            item_field,
+            OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 3, 3, 5])),
+            Arc::new(Int32Array::from(vec![
+                Some(1),
+                None,
+                Some(3),
+                Some(4),
+                Some(5),
+            ])),
+            Some(NullBuffer::from(vec![true, false, true, true, true, true])),
+        )
+        .unwrap();
+        let batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![
+                    Some(10),
+                    None,
+                    Some(30),
+                    Some(40),
+                    None,
+                    Some(60),
+                ])),
+                Arc::new(list),
+            ],
+        )
+        .unwrap();
+        let input = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema);
+        write_lance_file(
+            input,
+            &fs,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_3),
+                ..Default::default()
+            },
+        )
+        .await;
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let file_reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(file_reader.metadata.column_infos.len(), 2);
+        assert!(
+            file_reader
+                .metadata
+                .column_infos
+                .iter()
+                .flat_map(|column| column.page_infos.iter())
+                .all(|page| {
+                    matches!(
+                        &page.encoding,
+                        PageEncoding::Structural(layout)
+                            if matches!(
+                                layout.layout,
+                                Some(pb21::page_layout::Layout::SparseLayout(_))
+                            )
+                    )
+                })
+        );
+
+        let scan = file_reader
+            .read_stream(
+                lance_io::ReadBatchParams::RangeFull,
+                1024,
+                1,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(scan, vec![batch.clone()]);
+
+        let range = file_reader
+            .read_stream(
+                lance_io::ReadBatchParams::Range(1..5),
+                1024,
+                1,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(range, vec![batch.slice(1, 4)]);
+
+        let indices = UInt32Array::from(vec![0, 3, 5]);
+        let take = file_reader
+            .read_stream(
+                lance_io::ReadBatchParams::Indices(indices.clone()),
+                1024,
+                1,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(take, vec![batch.take(&indices).unwrap()]);
+    }
+
     async fn create_some_file(fs: &FsFixture, version: LanceFileVersion) -> WrittenFile {
         let location_type = DataType::Struct(Fields::from(vec![
             Field::new("x", DataType::Float64, true),
@@ -2660,6 +2923,60 @@ mod tests {
         .await
     }
 
+    async fn create_wide_fixed_size_list_file(fs: &FsFixture, num_columns: usize) -> WrittenFile {
+        let data_type =
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4);
+        let mut reader = gen_batch();
+        for column_idx in 0..num_columns {
+            reader = reader.col(
+                format!("c{column_idx}"),
+                array::rand_type(&data_type).with_random_nulls(0.1),
+            );
+        }
+        let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4));
+
+        write_lance_file(
+            reader,
+            fs,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    async fn create_wide_structural_file(fs: &FsFixture, num_groups: usize) -> WrittenFile {
+        let struct_type = DataType::Struct(Fields::from(vec![
+            Field::new("x", DataType::Int32, true),
+            Field::new("y", DataType::Int32, true),
+        ]));
+        let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+        let mut reader = gen_batch();
+        for group_idx in 0..num_groups {
+            reader = reader
+                .col(
+                    format!("s{group_idx}"),
+                    array::rand_type(&struct_type).with_random_nulls(0.5),
+                )
+                .col(
+                    format!("l{group_idx}"),
+                    array::rand_type(&list_type).with_random_nulls(0.5),
+                );
+        }
+        let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4));
+
+        write_lance_file(
+            reader,
+            fs,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
     type Transformer = Box<dyn Fn(&RecordBatch) -> RecordBatch>;
 
     async fn verify_expected(
@@ -3157,6 +3474,170 @@ mod tests {
         );
     }
 
+    async fn assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+        fs: &FsFixture,
+        projection: ReaderProjection,
+        shape: &str,
+    ) -> Vec<RecordBatch> {
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let eager_reader = FileReader::try_open(
+            file_scheduler.clone(),
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let expected = eager_reader
+            .read_stream_projected(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                16,
+                projection.clone(),
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let cache = test_cache();
+        let lazy_reader = ProjectedFileReader::try_open(
+            file_scheduler,
+            Some(projection.clone()),
+            Arc::<DecoderPlugins>::default(),
+            &cache,
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let metadata_index = lazy_reader.metadata_index().unwrap();
+        let requested_metadata_bytes = projection
+            .column_indices
+            .iter()
+            .map(|column_index| metadata_index.column_metadata_offsets[*column_index as usize].1)
+            .sum::<u64>();
+        let total_metadata_bytes = metadata_index
+            .column_metadata_offsets
+            .iter()
+            .map(|(_, length)| *length)
+            .sum::<u64>();
+        assert!(total_metadata_bytes > requested_metadata_bytes * 8);
+
+        fs.object_store.io_stats_incremental();
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::Range(0..0),
+                127,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        assert!(collect_read_tasks(tasks, 1).await.is_empty());
+        let metadata_stats = fs.object_store.io_stats_incremental();
+        assert!(
+            metadata_stats.read_bytes < total_metadata_bytes / 2,
+            "lazy {shape} read fetched too much metadata: read {} bytes, requested column metadata is {} bytes, total column metadata is {} bytes",
+            metadata_stats.read_bytes,
+            requested_metadata_bytes,
+            total_metadata_bytes
+        );
+
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let actual = collect_read_tasks(tasks, 16).await;
+        assert_eq!(expected, actual);
+        actual
+    }
+
+    #[tokio::test]
+    async fn test_lazy_reader_fixed_size_list_projection_matches_eager_reader() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_fixed_size_list_file(&fs, 512).await;
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["c17", "c509"],
+        )
+        .unwrap();
+        assert!(ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
+        assert!(!ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_0
+        ));
+        assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+            &fs,
+            projection,
+            "fixed-size-list",
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn test_lazy_reader_nested_projection_compacts_physical_columns() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_structural_file(&fs, 128).await;
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["s97.y", "l4", "s3"],
+        )
+        .unwrap();
+
+        assert_eq!(
+            projection
+                .schema
+                .fields
+                .iter()
+                .map(|field| field.name.as_str())
+                .collect::<Vec<_>>(),
+            vec!["s97", "l4", "s3"]
+        );
+        assert_eq!(projection.schema.fields[0].children.len(), 1);
+        assert_eq!(projection.schema.fields[0].children[0].name, "y");
+        assert_eq!(projection.schema.fields[2].children.len(), 2);
+        assert_eq!(projection.column_indices.len(), 4);
+        assert!(
+            projection
+                .column_indices
+                .windows(2)
+                .any(|indices| indices[0] > indices[1]),
+            "the projection must reorder physical columns to exercise compact remapping"
+        );
+        assert!(ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
+        let actual = assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+            &fs, projection, "nested",
+        )
+        .await;
+        assert!(
+            actual
+                .iter()
+                .flat_map(|batch| batch.columns())
+                .any(|column| column.null_count() > 0),
+            "the structural projection must exercise nullable arrays"
+        );
+    }
+
     #[rstest]
     #[case::before_metadata_region(90, 5)]
     #[case::after_metadata_region(190, 20)]
@@ -3185,19 +3666,23 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::blob(BLOB_META_KEY)]
+    #[case::packed_struct("lance-encoding:packed")]
     #[tokio::test]
-    async fn test_lazy_reader_rejects_unsupported_projection() {
+    async fn test_lazy_reader_rejects_opaque_projection(#[case] metadata_key: &str) {
         let fs = FsFixture::default();
         let written_file = create_some_file(&fs, LanceFileVersion::V2_1).await;
 
-        let projection = ReaderProjection::from_column_names(
+        let ordinary_projection = ReaderProjection::from_column_names(
             LanceFileVersion::V2_1,
             &written_file.schema,
-            &["location"],
+            &["location.x"],
         )
         .unwrap();
-        assert!(!ProjectedFileReader::supports_projection(
-            &projection,
+        assert_eq!(ordinary_projection.schema.fields[0].children.len(), 1);
+        assert!(ProjectedFileReader::supports_projection(
+            &ordinary_projection,
             LanceFileVersion::V2_1
         ));
 
@@ -3220,6 +3705,15 @@ mod tests {
             "expected InvalidInput, got {err:?}"
         );
 
+        let mut projection = ordinary_projection;
+        Arc::make_mut(&mut projection.schema).fields[0]
+            .metadata
+            .insert(metadata_key.to_string(), "true".to_string());
+        assert!(!ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
+
         let err = ProjectedFileReader::try_open(
             file_scheduler,
             Some(projection),
@@ -3231,7 +3725,7 @@ mod tests {
         .unwrap_err();
         assert!(
             matches!(err, lance_core::Error::NotSupported { .. }),
-            "expected NotSupported, got {err:?}"
+            "expected NotSupported for {metadata_key}, got {err:?}"
         );
     }
 
diff --git a/rust/lance-index-core/Cargo.toml b/rust/lance-index-core/Cargo.toml
new file mode 100644
index 00000000000..5024f2eee0b
--- /dev/null
+++ b/rust/lance-index-core/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "lance-index-core"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+readme = "README.md"
+description = "Core traits and types for Lance index plugins"
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+async-trait.workspace = true
+arrow-array.workspace = true
+arrow-schema.workspace = true
+arrow-select.workspace = true
+bytes.workspace = true
+datafusion-common.workspace = true
+datafusion-expr.workspace = true
+datafusion.workspace = true
+futures.workspace = true
+lance-core.workspace = true
+lance-io.workspace = true
+lance-select.workspace = true
+prost-types.workspace = true
+roaring.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+
+[lints]
+workspace = true
diff --git a/rust/lance-index-core/README.md b/rust/lance-index-core/README.md
new file mode 100644
index 00000000000..a954522a283
--- /dev/null
+++ b/rust/lance-index-core/README.md
@@ -0,0 +1,6 @@
+# lance-index-core
+
+`lance-index-core` is an internal sub-crate, containing the core traits and types used to
+implement index plugins for [Lance](https://github.com/lance-format/lance).
+
+**Important Note**: This crate is **not intended for external usage**.
diff --git a/rust/lance-index-core/src/lib.rs b/rust/lance-index-core/src/lib.rs
new file mode 100644
index 00000000000..393064014e4
--- /dev/null
+++ b/rust/lance-index-core/src/lib.rs
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{any::Any, sync::Arc};
+
+use async_trait::async_trait;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result};
+use roaring::RoaringBitmap;
+use serde::{Deserialize, Serialize};
+use std::convert::TryFrom;
+
+pub mod metrics;
+pub mod scalar;
+
+/// Generic methods common across all types of secondary indices
+///
+#[async_trait]
+pub trait Index: Send + Sync + DeepSizeOf {
+    /// Cast to [Any].
+    fn as_any(&self) -> &dyn Any;
+
+    /// Cast to [Index]
+    fn as_index(self: Arc<Self>) -> Arc<dyn Index>;
+
+    /// Retrieve index statistics as a JSON Value
+    fn statistics(&self) -> Result<serde_json::Value>;
+
+    /// Prewarm the index.
+    ///
+    /// This will load the index into memory and cache it.
+    async fn prewarm(&self) -> Result<()>;
+
+    /// Get the type of the index
+    fn index_type(&self) -> IndexType;
+
+    /// Read through the index and determine which fragment ids are covered by the index
+    ///
+    /// This is a kind of slow operation.  It's better to use the fragment_bitmap.  This
+    /// only exists for cases where the fragment_bitmap has become corrupted or missing.
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap>;
+}
+
+/// Index Type
+#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf, Serialize, Deserialize)]
+pub enum IndexType {
+    // Preserve 0-100 for simple indices.
+    Scalar = 0, // Legacy scalar index, alias to BTree
+
+    BTree = 1, // BTree
+
+    Bitmap = 2, // Bitmap
+
+    LabelList = 3, // LabelList
+
+    Inverted = 4, // Inverted
+
+    NGram = 5, // NGram
+
+    FragmentReuse = 6,
+
+    MemWal = 7,
+
+    ZoneMap = 8, // ZoneMap
+
+    BloomFilter = 9, // Bloom filter
+
+    RTree = 10, // RTree
+
+    Fm = 11, // FM-Index
+
+    // 100+ and up for vector index.
+    /// Flat vector index.
+    Vector = 100, // Legacy vector index, alias to IvfPq
+    IvfFlat = 101,
+    IvfSq = 102,
+    IvfPq = 103,
+    IvfHnswSq = 104,
+    IvfHnswPq = 105,
+    IvfHnswFlat = 106,
+    IvfRq = 107,
+}
+
+impl std::fmt::Display for IndexType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::Scalar | Self::BTree => write!(f, "BTree"),
+            Self::Bitmap => write!(f, "Bitmap"),
+            Self::LabelList => write!(f, "LabelList"),
+            Self::Inverted => write!(f, "Inverted"),
+            Self::NGram => write!(f, "NGram"),
+            Self::FragmentReuse => write!(f, "FragmentReuse"),
+            Self::MemWal => write!(f, "MemWal"),
+            Self::ZoneMap => write!(f, "ZoneMap"),
+            Self::BloomFilter => write!(f, "BloomFilter"),
+            Self::RTree => write!(f, "RTree"),
+            Self::Fm => write!(f, "Fm"),
+            Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"),
+            Self::IvfFlat => write!(f, "IVF_FLAT"),
+            Self::IvfSq => write!(f, "IVF_SQ"),
+            Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
+            Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
+            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
+            Self::IvfRq => write!(f, "IVF_RQ"),
+        }
+    }
+}
+
+impl TryFrom<i32> for IndexType {
+    type Error = Error;
+
+    fn try_from(value: i32) -> Result<Self> {
+        match value {
+            v if v == Self::Scalar as i32 => Ok(Self::Scalar),
+            v if v == Self::BTree as i32 => Ok(Self::BTree),
+            v if v == Self::Bitmap as i32 => Ok(Self::Bitmap),
+            v if v == Self::LabelList as i32 => Ok(Self::LabelList),
+            v if v == Self::NGram as i32 => Ok(Self::NGram),
+            v if v == Self::Inverted as i32 => Ok(Self::Inverted),
+            v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse),
+            v if v == Self::MemWal as i32 => Ok(Self::MemWal),
+            v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap),
+            v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter),
+            v if v == Self::RTree as i32 => Ok(Self::RTree),
+            v if v == Self::Fm as i32 => Ok(Self::Fm),
+            v if v == Self::Vector as i32 => Ok(Self::Vector),
+            v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat),
+            v if v == Self::IvfSq as i32 => Ok(Self::IvfSq),
+            v if v == Self::IvfPq as i32 => Ok(Self::IvfPq),
+            v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq),
+            v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq),
+            v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat),
+            v if v == Self::IvfRq as i32 => Ok(Self::IvfRq),
+            _ => Err(Error::invalid_input_source(
+                format!("the input value {} is not a valid IndexType", value).into(),
+            )),
+        }
+    }
+}
+
+impl TryFrom<&str> for IndexType {
+    type Error = Error;
+
+    fn try_from(value: &str) -> Result<Self> {
+        match value {
+            "BTree" | "BTREE" => Ok(Self::BTree),
+            "Bitmap" | "BITMAP" => Ok(Self::Bitmap),
+            "LabelList" | "LABELLIST" => Ok(Self::LabelList),
+            "Inverted" | "INVERTED" => Ok(Self::Inverted),
+            "NGram" | "NGRAM" => Ok(Self::NGram),
+            "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap),
+            "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter),
+            "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree),
+            "Fm" | "FM" => Ok(Self::Fm),
+            "Vector" | "VECTOR" => Ok(Self::Vector),
+            "IVF_FLAT" => Ok(Self::IvfFlat),
+            "IVF_SQ" => Ok(Self::IvfSq),
+            "IVF_PQ" => Ok(Self::IvfPq),
+            "IVF_RQ" => Ok(Self::IvfRq),
+            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
+            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
+            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
+            "FragmentReuse" => Ok(Self::FragmentReuse),
+            "MemWal" => Ok(Self::MemWal),
+            _ => Err(Error::invalid_input(format!(
+                "invalid index type: {}",
+                value
+            ))),
+        }
+    }
+}
+
+impl IndexType {
+    pub fn is_scalar(&self) -> bool {
+        matches!(
+            self,
+            Self::Scalar
+                | Self::BTree
+                | Self::Bitmap
+                | Self::LabelList
+                | Self::Inverted
+                | Self::NGram
+                | Self::ZoneMap
+                | Self::BloomFilter
+                | Self::RTree
+                | Self::Fm,
+        )
+    }
+
+    pub fn is_vector(&self) -> bool {
+        matches!(
+            self,
+            Self::Vector
+                | Self::IvfPq
+                | Self::IvfHnswSq
+                | Self::IvfHnswPq
+                | Self::IvfHnswFlat
+                | Self::IvfFlat
+                | Self::IvfSq
+                | Self::IvfRq
+        )
+    }
+
+    pub fn is_system(&self) -> bool {
+        matches!(self, Self::FragmentReuse | Self::MemWal)
+    }
+
+    /// Returns the current format version of the index type,
+    /// bump this when the index format changes.
+    /// Indices which higher version than these will be ignored for compatibility,
+    /// This would happen when creating index in a newer version of Lance,
+    /// but then opening the index in older version of Lance
+    pub fn version(&self) -> i32 {
+        match self {
+            Self::Scalar => 0,
+            Self::BTree => 0,
+            Self::Bitmap => 0,
+            Self::LabelList => 0,
+            Self::Inverted => 0,
+            Self::NGram => 0,
+            Self::FragmentReuse => 0,
+            Self::MemWal => 0,
+            Self::ZoneMap => 0,
+            Self::BloomFilter => 0,
+            Self::RTree => 0,
+            Self::Fm => 0,
+
+            // IMPORTANT: if any vector index subtype needs a format bump that is
+            // not backward compatible, its new version must be set to
+            // (current max vector index version + 1), even if only one subtype
+            // changed. Compatibility filtering currently cannot distinguish vector
+            // subtypes from details-only metadata, so vector versions effectively
+            // share one global monotonic compatibility level.
+            Self::Vector
+            | Self::IvfFlat
+            | Self::IvfSq
+            | Self::IvfPq
+            | Self::IvfHnswSq
+            | Self::IvfHnswPq
+            | Self::IvfHnswFlat => 1,
+            Self::IvfRq => 2,
+        }
+    }
+
+    /// Returns the target partition size for the index type.
+    ///
+    /// This is used to compute the number of partitions for the index.
+    /// The partition size is optimized for the best performance of the index.
+    ///
+    /// This is for vector indices only.
+    pub fn target_partition_size(&self) -> usize {
+        match self {
+            Self::Vector => 8192,
+            Self::IvfFlat => 4096,
+            Self::IvfSq => 8192,
+            Self::IvfPq => 8192,
+            Self::IvfRq => 4096,
+            Self::IvfHnswFlat => 1 << 20,
+            Self::IvfHnswSq => 1 << 20,
+            Self::IvfHnswPq => 1 << 20,
+            _ => 8192,
+        }
+    }
+
+    /// Returns the highest supported vector index version in this Lance build.
+    pub fn max_vector_version() -> u32 {
+        [
+            Self::Vector,
+            Self::IvfFlat,
+            Self::IvfSq,
+            Self::IvfPq,
+            Self::IvfHnswSq,
+            Self::IvfHnswPq,
+            Self::IvfHnswFlat,
+            Self::IvfRq,
+        ]
+        .into_iter()
+        .map(|index_type| index_type.version() as u32)
+        .max()
+        .unwrap_or(1)
+    }
+}
+
+pub trait IndexParams: Send + Sync {
+    fn as_any(&self) -> &dyn Any;
+
+    fn index_name(&self) -> &str;
+}
diff --git a/rust/lance-index-core/src/metrics.rs b/rust/lance-index-core/src/metrics.rs
new file mode 100644
index 00000000000..8c0c119a3c3
--- /dev/null
+++ b/rust/lance-index-core/src/metrics.rs
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen";
+pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return";
+pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores";
+pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected";
+
+/// A trait used by the index to report metrics
+///
+/// Callers can implement this trait to collect metrics
+pub trait MetricsCollector: Send + Sync {
+    /// Record partition loads
+    ///
+    /// Many indices consist of partitions that may need to be loaded
+    /// into cache.  For example, an inverted index or ngram index has a
+    /// posting list for each token.
+    ///
+    /// In the ideal case, these shards are in the cache and will not need
+    /// to be loaded from disk.  This method should not be called if the
+    /// shard is in the cache.
+    fn record_parts_loaded(&self, num_parts: usize);
+
+    /// Record a shard load
+    fn record_part_load(&self) {
+        self.record_parts_loaded(1);
+    }
+
+    /// Record an index load
+    ///
+    /// This should be called when a scalar index is loaded from storage.
+    /// It should not be called if the index is already in memory.
+    fn record_index_loads(&self, num_indexes: usize);
+
+    /// Record an index load
+    fn record_index_load(&self) {
+        self.record_index_loads(1);
+    }
+
+    /// Record the number of "comparisons" made by the index
+    ///
+    /// What exactly constitutes a comparison depends on the index type.
+    /// For example, a B-tree index may make comparisons while searching for a value.
+    /// On the other hand, a bitmap index makes comparisons when computing the intersection
+    /// of two bitmaps.
+    ///
+    /// The goal is to provide some visibility into the compute cost of the search
+    fn record_comparisons(&self, num_comparisons: usize);
+
+    /// Record AND candidates returned from WAND alignment to the scoring loop.
+    ///
+    /// This excludes candidates pruned before `next()` returns. Use this with
+    /// `record_and_candidates_pruned_before_return` to recover total aligned
+    /// AND candidates.
+    fn record_and_candidates_seen(&self, _num_candidates: usize) {}
+
+    /// Record AND candidates pruned during WAND alignment before `next()` returns.
+    fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {}
+
+    fn record_and_full_scores(&self, _num_scores: usize) {}
+
+    fn record_freqs_collected(&self, _num_collections: usize) {}
+
+    /// Returns an optional sink for recording exact I/O statistics (bytes read,
+    /// IOPS, and requests) performed on behalf of this collector.
+    ///
+    /// Index implementations that read from a
+    /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to
+    /// their file readers so the I/O performed for a single query is measured
+    /// and attributed here.  The default returns `None`, meaning the caller does
+    /// not want I/O measured (and index implementations should then take their
+    /// normal, uninstrumented read path).
+    fn io_stats(&self) -> Option<lance_io::scheduler::IoStats> {
+        None
+    }
+}
+
+/// A no-op metrics collector that does nothing
+pub struct NoOpMetricsCollector;
+
+impl MetricsCollector for NoOpMetricsCollector {
+    fn record_parts_loaded(&self, _num_parts: usize) {}
+    fn record_index_loads(&self, _num_indexes: usize) {}
+    fn record_comparisons(&self, _num_comparisons: usize) {}
+}
+
+#[derive(Default)]
+pub struct LocalMetricsCollector {
+    pub parts_loaded: AtomicUsize,
+    pub index_loads: AtomicUsize,
+    pub comparisons: AtomicUsize,
+}
+
+impl LocalMetricsCollector {
+    pub fn dump_into(self, other: &dyn MetricsCollector) {
+        other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed));
+        other.record_index_loads(self.index_loads.load(Ordering::Relaxed));
+        other.record_comparisons(self.comparisons.load(Ordering::Relaxed));
+    }
+}
+
+impl MetricsCollector for LocalMetricsCollector {
+    fn record_parts_loaded(&self, num_parts: usize) {
+        self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed);
+    }
+
+    fn record_index_loads(&self, num_indexes: usize) {
+        self.index_loads.fetch_add(num_indexes, Ordering::Relaxed);
+    }
+
+    fn record_comparisons(&self, num_comparisons: usize) {
+        self.comparisons
+            .fetch_add(num_comparisons, Ordering::Relaxed);
+    }
+}
diff --git a/rust/lance-index-core/src/scalar.rs b/rust/lance-index-core/src/scalar.rs
new file mode 100644
index 00000000000..e209a32cdb8
--- /dev/null
+++ b/rust/lance-index-core/src/scalar.rs
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Abstract scalar index traits and types for Lance index plugins
+
+use arrow_array::{BooleanArray, RecordBatch, UInt64Array};
+use arrow_schema::Schema;
+use async_trait::async_trait;
+use bytes::Bytes;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion_common::scalar::ScalarValue;
+use datafusion_expr::Expr;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use lance_core::{Error, Result};
+use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter};
+use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
+use roaring::{RoaringBitmap, RoaringTreemap};
+use serde::Serialize;
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::pin::Pin;
+use std::{any::Any, sync::Arc};
+
+use crate::metrics::MetricsCollector;
+use crate::{Index, IndexParams, IndexType};
+
+/// Metadata about a single file within an index segment.
+#[derive(Debug, Clone, PartialEq, DeepSizeOf)]
+pub struct IndexFile {
+    /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx")
+    pub path: String,
+    /// Size of the file in bytes
+    pub size_bytes: u64,
+}
+
+pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index";
+
+/// Builtin index types supported by the Lance library
+///
+/// This is primarily for convenience to avoid a bunch of string
+/// constants and provide some auto-complete.  This type should not
+/// be used in the manifest as plugins cannot add new entries.
+#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
+pub enum BuiltinIndexType {
+    BTree,
+    Bitmap,
+    LabelList,
+    NGram,
+    ZoneMap,
+    BloomFilter,
+    RTree,
+    Inverted,
+    Fm,
+}
+
+impl BuiltinIndexType {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::BTree => "btree",
+            Self::Bitmap => "bitmap",
+            Self::LabelList => "labellist",
+            Self::NGram => "ngram",
+            Self::ZoneMap => "zonemap",
+            Self::Inverted => "inverted",
+            Self::BloomFilter => "bloomfilter",
+            Self::RTree => "rtree",
+            Self::Fm => "fm",
+        }
+    }
+}
+
+impl TryFrom<IndexType> for BuiltinIndexType {
+    type Error = Error;
+
+    fn try_from(value: IndexType) -> Result<Self> {
+        match value {
+            IndexType::BTree => Ok(Self::BTree),
+            IndexType::Bitmap => Ok(Self::Bitmap),
+            IndexType::LabelList => Ok(Self::LabelList),
+            IndexType::NGram => Ok(Self::NGram),
+            IndexType::ZoneMap => Ok(Self::ZoneMap),
+            IndexType::Inverted => Ok(Self::Inverted),
+            IndexType::BloomFilter => Ok(Self::BloomFilter),
+            IndexType::RTree => Ok(Self::RTree),
+            IndexType::Fm => Ok(Self::Fm),
+            _ => Err(Error::index("Invalid index type".to_string())),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct ScalarIndexParams {
+    /// The type of index to create
+    ///
+    /// Plugins may add additional index types.  Index type lookup is case-insensitive.
+    pub index_type: String,
+    /// The parameters to train the index
+    ///
+    /// This should be a JSON string.  The contents of the JSON string will be specific to the
+    /// index type.  If not set, then default parameters will be used for the index type.
+    pub params: Option<String>,
+}
+
+impl Default for ScalarIndexParams {
+    fn default() -> Self {
+        Self {
+            index_type: BuiltinIndexType::BTree.as_str().to_string(),
+            params: None,
+        }
+    }
+}
+
+impl ScalarIndexParams {
+    /// Creates a new ScalarIndexParams from one of the builtin index types
+    pub fn for_builtin(index_type: BuiltinIndexType) -> Self {
+        Self {
+            index_type: index_type.as_str().to_string(),
+            params: None,
+        }
+    }
+
+    /// Create a new ScalarIndexParams with the given index type
+    pub fn new(index_type: String) -> Self {
+        Self {
+            index_type,
+            params: None,
+        }
+    }
+
+    /// Set the parameters for the index
+    pub fn with_params<ParamsType: Serialize>(mut self, params: &ParamsType) -> Self {
+        self.params = Some(serde_json::to_string(params).unwrap());
+        self
+    }
+}
+
+impl IndexParams for ScalarIndexParams {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn index_name(&self) -> &str {
+        LANCE_SCALAR_INDEX
+    }
+}
+
+/// Trait for storing an index (or parts of an index) into storage
+#[async_trait]
+pub trait IndexWriter: Send {
+    /// Writes a record batch into the file, returning the 0-based index of the batch in the file
+    ///
+    /// E.g. if this is the third time this is called this method will return 2
+    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64>;
+    /// Adds a global buffer and returns its index.
+    async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index writer",
+        ))
+    }
+    /// Finishes writing the file and closes the file
+    async fn finish(&mut self) -> Result<IndexFile>;
+    /// Finishes writing the file and closes the file with additional metadata
+    async fn finish_with_metadata(
+        &mut self,
+        metadata: HashMap<String, String>,
+    ) -> Result<IndexFile>;
+}
+
+/// Trait for reading an index (or parts of an index) from storage
+#[async_trait]
+pub trait IndexReader: Send + Sync {
+    /// Read the n-th record batch from the file
+    async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch>;
+    /// Reads a global buffer by index.
+    async fn read_global_buffer(&self, _index: u32) -> Result<Bytes> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index reader",
+        ))
+    }
+    /// Read the range of rows from the file.
+    /// If projection is Some, only return the columns in the projection,
+    /// nested columns like Some(&["x.y"]) are not supported.
+    /// If projection is None, return all columns.
+    async fn read_range(
+        &self,
+        range: std::ops::Range<usize>,
+        projection: Option<&[&str]>,
+    ) -> Result<RecordBatch>;
+    /// Read multiple ranges and concatenate into a single batch.
+    /// Default impl runs `read_range`s in parallel via `try_join_all`.
+    async fn read_ranges(
+        &self,
+        ranges: &[std::ops::Range<usize>],
+        projection: Option<&[&str]>,
+    ) -> Result<RecordBatch> {
+        if ranges.is_empty() {
+            return self.read_range(0..0, projection).await;
+        }
+        let futures = ranges
+            .iter()
+            .map(|r| self.read_range(r.clone(), projection));
+        let batches = futures::future::try_join_all(futures).await?;
+        let schema = batches[0].schema();
+        Ok(arrow_select::concat::concat_batches(&schema, &batches)?)
+    }
+    /// Read a range of rows as a stream of record batches.
+    ///
+    /// This allows the caller to process rows incrementally without loading the
+    /// entire range into memory at once.
+    ///
+    /// The default implementation falls back to [`Self::read_range`] and wraps
+    /// the result in a single-item stream.
+    async fn read_range_stream(
+        &self,
+        range: std::ops::Range<usize>,
+        projection: Option<&[&str]>,
+    ) -> Result<Pin<Box<dyn RecordBatchStream>>> {
+        let batch = self.read_range(range, projection).await?;
+        let schema = batch.schema();
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move { Ok(batch) }),
+        )))
+    }
+    /// Return the number of batches in the file
+    async fn num_batches(&self, batch_size: u64) -> u32;
+    /// Return the number of rows in the file
+    fn num_rows(&self) -> usize;
+    /// Return the metadata of the file
+    fn schema(&self) -> &lance_core::datatypes::Schema;
+    /// Best-effort on-disk byte size of the file when the reader already knows it
+    /// without extra I/O, else `None`. Used to size prewarm chunks.
+    fn file_size_bytes(&self) -> Option<u64> {
+        None
+    }
+}
+
+/// Trait abstracting I/O away from index logic
+///
+/// Scalar indices are currently serialized as indexable arrow record batches stored in
+/// named "files".  The index store is responsible for serializing and deserializing
+/// these batches into file data (e.g. as .lance files or .parquet files, etc.)
+#[async_trait]
+pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
+    fn as_any(&self) -> &dyn Any;
+    fn clone_arc(&self) -> Arc<dyn IndexStore>;
+
+    /// Suggested I/O parallelism for the store
+    fn io_parallelism(&self) -> usize;
+
+    /// Create a new file and return a writer to store data in the file
+    async fn new_index_file(&self, name: &str, schema: Arc<Schema>)
+    -> Result<Box<dyn IndexWriter>>;
+
+    /// Open an existing file for retrieval
+    async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>>;
+
+    /// Return a store that submits its I/O at the given base priority.
+    fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore>;
+
+    /// Copy a range of batches from an index file from this store to another
+    ///
+    /// This is often useful when remapping or updating
+    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<IndexFile>;
+
+    /// Copy an index file from this store to a new name in another store, leaving the source intact
+    async fn copy_index_file_to(
+        &self,
+        name: &str,
+        new_name: &str,
+        dest_store: &dyn IndexStore,
+    ) -> Result<IndexFile> {
+        if name == new_name {
+            self.copy_index_file(name, dest_store).await
+        } else {
+            Err(Error::not_supported(format!(
+                "copying index file {name} to {new_name} is not supported by this index store"
+            )))
+        }
+    }
+
+    /// Rename an index file
+    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile>;
+
+    /// Delete an index file (used in the tmp spill store to keep tmp size down)
+    async fn delete_index_file(&self, name: &str) -> Result<()>;
+
+    /// List all files in the index directory with their sizes.
+    ///
+    /// Returns a list of (relative_path, size_bytes) tuples.
+    /// Used to capture file metadata after index creation/modification.
+    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>>;
+}
+
+/// Different scalar indices may support different kinds of queries
+///
+/// For example, a btree index can support a wide range of queries (e.g. x > 7)
+/// while an index based on FTS only supports queries like "x LIKE 'foo'"
+///
+/// This trait is used when we need an object that can represent any kind of query
+///
+/// Note: if you are implementing this trait for a query type then you probably also
+/// need to implement the scalar query parser trait to create instances of your query at parse time.
+pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync {
+    /// Cast the query as Any to allow for downcasting
+    fn as_any(&self) -> &dyn Any;
+    /// Format the query as a string for display purposes
+    fn format(&self, col: &str) -> String;
+    /// Convert the query to a datafusion expression
+    fn to_expr(&self, col: String) -> Expr;
+    /// Compare this query to another query
+    fn dyn_eq(&self, other: &dyn AnyQuery) -> bool;
+}
+
+impl PartialEq for dyn AnyQuery {
+    fn eq(&self, other: &Self) -> bool {
+        self.dyn_eq(other)
+    }
+}
+
+/// The result of a search operation against a scalar index
+#[derive(Debug, PartialEq)]
+pub enum SearchResult {
+    /// The exact row ids that satisfy the query
+    Exact(NullableRowAddrSet),
+    /// Any row id satisfying the query will be in this set but not every
+    /// row id in this set will satisfy the query, a further recheck step
+    /// is needed
+    AtMost(NullableRowAddrSet),
+    /// All of the given row ids satisfy the query but there may be more
+    ///
+    /// No scalar index actually returns this today but it can arise from
+    /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x)))
+    AtLeast(NullableRowAddrSet),
+}
+
+impl SearchResult {
+    pub fn exact(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn at_most(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn at_least(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn with_nulls(self, nulls: impl Into<RowAddrTreeMap>) -> Self {
+        match self {
+            Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())),
+            Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())),
+            Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())),
+        }
+    }
+
+    pub fn row_addrs(&self) -> &NullableRowAddrSet {
+        match self {
+            Self::Exact(row_addrs) => row_addrs,
+            Self::AtMost(row_addrs) => row_addrs,
+            Self::AtLeast(row_addrs) => row_addrs,
+        }
+    }
+
+    pub fn is_exact(&self) -> bool {
+        matches!(self, Self::Exact(_))
+    }
+}
+
+/// Brief information about an index that was created
+pub struct CreatedIndex {
+    /// The details of the index that was created
+    ///
+    /// These should be stored somewhere as they will be needed to
+    /// load the index later.
+    pub index_details: prost_types::Any,
+    /// The version of the index that was created
+    ///
+    /// This can be used to determine if a reader is able to load the index.
+    pub index_version: u32,
+    /// List of files and their sizes for this index
+    ///
+    /// This enables skipping HEAD calls when opening indices and provides
+    /// visibility into index storage size via describe_indices().
+    pub files: Vec<IndexFile>,
+}
+
+/// The ordering that training data must satisfy
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrainingOrdering {
+    /// The input will arrive sorted by the value column in ascending order
+    Values,
+    /// The input will arrive sorted by the address column in ascending order
+    Addresses,
+    /// The input will arrive in an arbitrary order
+    None,
+}
+
+#[derive(Debug, Clone)]
+pub struct TrainingCriteria {
+    pub ordering: TrainingOrdering,
+    pub needs_row_ids: bool,
+    pub needs_row_addrs: bool,
+}
+
+impl TrainingCriteria {
+    pub fn new(ordering: TrainingOrdering) -> Self {
+        Self {
+            ordering,
+            needs_row_ids: false,
+            needs_row_addrs: false,
+        }
+    }
+
+    pub fn with_row_id(mut self) -> Self {
+        self.needs_row_ids = true;
+        self
+    }
+
+    pub fn with_row_addr(mut self) -> Self {
+        self.needs_row_addrs = true;
+        self
+    }
+}
+
+/// The criteria that specifies how to update an index
+pub struct UpdateCriteria {
+    /// If true, then we need to read the old data to update the index
+    ///
+    /// This should be avoided if possible but is left in for some legacy paths
+    pub requires_old_data: bool,
+    /// The criteria required for data (both old and new)
+    pub data_criteria: TrainingCriteria,
+}
+
+/// Filter used when merging existing scalar-index rows during update.
+///
+/// The caller must pick a filter mode that matches the row-id semantics of the
+/// dataset:
+/// - address-style row IDs: fragment filtering is valid
+/// - stable row IDs: use exact row-id membership instead
+#[derive(Debug, Clone)]
+pub enum OldIndexDataFilter {
+    /// Keeps track of which fragments are still valid and which are no longer valid.
+    ///
+    /// This is valid for address-style row IDs.
+    Fragments {
+        to_keep: RoaringBitmap,
+        to_remove: RoaringBitmap,
+    },
+    /// Keep old rows whose row IDs are in this exact allow-list.
+    ///
+    /// This is required for stable row IDs, where row IDs are opaque and
+    /// should not be interpreted as encoded row addresses.
+    RowIds(RowAddrTreeMap),
+}
+
+impl OldIndexDataFilter {
+    /// Build a boolean mask that keeps only row IDs selected by this filter.
+    pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray {
+        match self {
+            Self::Fragments { to_keep, .. } => row_ids
+                .iter()
+                .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32)))
+                .collect(),
+            Self::RowIds(valid_row_ids) => row_ids
+                .iter()
+                .map(|id| id.map(|id| valid_row_ids.contains(id)))
+                .collect(),
+        }
+    }
+
+    /// Apply this filter in place to a set of existing (old) row ids/addresses,
+    /// retaining only the rows the filter selects to keep. Used by index types
+    /// that merge old postings directly (e.g. bitmap) instead of re-scanning a
+    /// row-id array through [`Self::filter_row_ids`].
+    pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) {
+        match self {
+            Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()),
+            Self::RowIds(valid_row_ids) => *rows &= valid_row_ids,
+        }
+    }
+}
+
+impl UpdateCriteria {
+    pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self {
+        Self {
+            requires_old_data: true,
+            data_criteria,
+        }
+    }
+
+    pub fn only_new_data(data_criteria: TrainingCriteria) -> Self {
+        Self {
+            requires_old_data: false,
+            data_criteria,
+        }
+    }
+}
+
+/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries
+#[async_trait]
+pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
+    /// Search the scalar index
+    ///
+    /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered
+    async fn search(
+        &self,
+        query: &dyn AnyQuery,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<SearchResult>;
+
+    /// Returns true if this index reports matches as physical row addresses
+    /// (`fragment_id << 32 | offset`) rather than row ids
+    ///
+    /// Address-domain indices (e.g. zone map, bloom filter) are built over the
+    /// `_rowaddr` column. On a dataset with stable row ids the address and
+    /// row-id domains diverge, so these results must be translated back to row
+    /// ids (via the per-fragment row-id sequences, known only at the dataset
+    /// layer) before they are combined with row-id results or handed to the
+    /// scan. The default (row-id domain) needs no translation.
+    fn results_are_row_addresses(&self) -> bool {
+        false
+    }
+
+    /// Returns true if the remap operation is supported
+    fn can_remap(&self) -> bool;
+
+    /// Remap the row ids, creating a new remapped version of this index in `dest_store`
+    async fn remap(
+        &self,
+        mapping: &RowAddrRemap,
+        dest_store: &dyn IndexStore,
+    ) -> Result<CreatedIndex>;
+
+    /// Add the new data into the index, creating an updated version of the index in `dest_store`
+    ///
+    /// If `old_data_filter` is provided, old index data will be filtered before
+    /// merge according to the chosen filter mode.
+    async fn update(
+        &self,
+        new_data: SendableRecordBatchStream,
+        dest_store: &dyn IndexStore,
+        old_data_filter: Option<OldIndexDataFilter>,
+    ) -> Result<CreatedIndex>;
+
+    /// Returns the criteria that will be used to update the index
+    fn update_criteria(&self) -> UpdateCriteria;
+
+    /// Derive the index parameters from the current index
+    ///
+    /// This returns a ScalarIndexParams that can be used to recreate an index
+    /// with the same configuration on another dataset.
+    fn derive_index_params(&self) -> Result<ScalarIndexParams>;
+
+    /// Global `[min, max]` of the indexed column from index metadata, without a
+    /// scan, or `None` if this index type cannot supply a sound bound. When
+    /// `Some`, the range is a superset of live values (conservative under
+    /// deletes): safe to prune with, not guaranteed tight.
+    fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> {
+        None
+    }
+}
+
+/// Abstraction over any type that can remap row IDs during index loading.
+///
+/// This decouples scalar index plugins from the table-level frag reuse index type.
+/// The frag reuse index implements this trait, but callers may also supply custom
+/// implementations for testing or other remapping strategies.
+pub trait RowIdRemapper: Send + Sync + std::fmt::Debug {
+    /// Remap a single row id.  Returns `None` if the row was deleted.
+    fn remap_row_id(&self, row_id: u64) -> Option<u64>;
+    /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows.
+    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap;
+    /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows.
+    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap;
+    /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows.
+    fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch>;
+}
diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml
index b9d1a5fde29..4387dc299f7 100644
--- a/rust/lance-index/Cargo.toml
+++ b/rust/lance-index/Cargo.toml
@@ -15,6 +15,7 @@ rust-version.workspace = true
 arc-swap.workspace = true
 arrow.workspace = true
 arrow-array.workspace = true
+arrow-ipc.workspace = true
 arrow-ord.workspace = true
 arrow-schema.workspace = true
 arrow-select.workspace = true
@@ -39,6 +40,7 @@ jsonb.workspace = true
 lance-arrow.workspace = true
 lance-arrow-stats.workspace = true
 lance-core.workspace = true
+lance-index-core.workspace = true
 lance-datafusion.workspace = true
 lance-encoding.workspace = true
 lance-file.workspace = true
diff --git a/rust/lance-index/benches/hnsw.rs b/rust/lance-index/benches/hnsw.rs
index 0a9b10bf42c..5884410ad9a 100644
--- a/rust/lance-index/benches/hnsw.rs
+++ b/rust/lance-index/benches/hnsw.rs
@@ -53,6 +53,7 @@ fn bench_hnsw(c: &mut Criterion) {
                         lower_bound: None,
                         upper_bound: None,
                         dist_q_c: 0.0,
+                        use_acorn: false,
                     },
                     None,
                     vectors.as_ref(),
@@ -81,6 +82,7 @@ fn bench_hnsw(c: &mut Criterion) {
                         lower_bound: None,
                         upper_bound: None,
                         dist_q_c: 0.0,
+                        use_acorn: false,
                     },
                     None,
                     vectors.as_ref(),
@@ -141,6 +143,7 @@ fn bench_hnsw_load(c: &mut Criterion) {
                             lower_bound: None,
                             upper_bound: None,
                             dist_q_c: 0.0,
+                            use_acorn: false,
                         },
                         None,
                         vectors.as_ref(),
@@ -206,6 +209,7 @@ fn bench_hnsw_sq(c: &mut Criterion) {
                             lower_bound: None,
                             upper_bound: None,
                             dist_q_c: 0.0,
+                            use_acorn: false,
                         },
                         None,
                         vectors.as_ref(),
@@ -235,6 +239,7 @@ fn bench_hnsw_sq(c: &mut Criterion) {
                         lower_bound: None,
                         upper_bound: None,
                         dist_q_c: 0.0,
+                        use_acorn: false,
                     },
                     None,
                     vectors.as_ref(),
@@ -302,6 +307,7 @@ fn bench_hnsw_pq(c: &mut Criterion) {
                             lower_bound: None,
                             upper_bound: None,
                             dist_q_c: 0.0,
+                            use_acorn: false,
                         },
                         None,
                         vectors.as_ref(),
@@ -331,6 +337,7 @@ fn bench_hnsw_pq(c: &mut Criterion) {
                         lower_bound: None,
                         upper_bound: None,
                         dist_q_c: 0.0,
+                        use_acorn: false,
                     },
                     None,
                     vectors.as_ref(),
diff --git a/rust/lance-index/examples/acorn_bench.rs b/rust/lance-index/examples/acorn_bench.rs
new file mode 100644
index 00000000000..8a24088e89b
--- /dev/null
+++ b/rust/lance-index/examples/acorn_bench.rs
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Compare the current mask-aware HNSW traversal (`search_basic`) against an
+//! ACORN-1 style traversal (`search_acorn`) and a flat scan over matching
+//! rows, across filter selectivities and mask shapes:
+//!
+//! - `corr-in`: mask is a cluster in embedding space, query is inside it
+//!   (e.g. "filter to courses" with a course query)
+//! - `corr-out`: mask is a cluster, query is unrelated
+//! - `random`: uniform random mask (the easy case)
+//!
+//! Run: cargo run --release -p lance-index --example acorn_bench
+
+#![allow(clippy::print_stdout)]
+
+use std::sync::Arc;
+use std::time::Instant;
+
+use arrow_array::{Array, FixedSizeListArray, types::Float32Type};
+use lance_arrow::FixedSizeListArrayExt;
+use lance_index::vector::flat::storage::FlatFloatStorage;
+use lance_index::vector::graph::VisitedGenerator;
+use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams};
+use lance_index::vector::storage::{DistCalculator, VectorStore};
+use lance_index::vector::v3::subindex::IvfSubIndex;
+use lance_linalg::distance::DistanceType;
+use lance_testing::datagen::generate_random_array_with_seed;
+use rand::rngs::SmallRng;
+use rand::seq::{IndexedRandom, SliceRandom};
+use rand::{Rng, SeedableRng};
+
+const TOTAL: usize = 100_000;
+const DIMENSION: usize = 768;
+const K: usize = 10;
+const EF: usize = 100;
+const EF_HI: usize = 400;
+const QUERIES_PER_CASE: usize = 20;
+const SEED: [u8; 32] = [42; 32];
+
+fn all_distances(storage: &FlatFloatStorage, query: Arc<dyn Array>) -> Vec<f32> {
+    let dist_calc = storage.dist_calculator(query, 0.0);
+    (0..TOTAL as u32).map(|i| dist_calc.distance(i)).collect()
+}
+
+/// Node ids sorted ascending by distance to `query`, restricted to `mask`.
+fn ground_truth(storage: &FlatFloatStorage, query: Arc<dyn Array>, mask: &[bool]) -> Vec<u32> {
+    let dists = all_distances(storage, query);
+    let mut ids: Vec<u32> = (0..TOTAL as u32).filter(|&i| mask[i as usize]).collect();
+    ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap());
+    ids.truncate(K);
+    ids
+}
+
+fn recall(got: &[u32], truth: &[u32]) -> f64 {
+    let hits = truth.iter().filter(|id| got.contains(id)).count();
+    hits as f64 / truth.len().max(1) as f64
+}
+
+struct CaseResult {
+    latency_us: Vec<f64>,
+    recalls: Vec<f64>,
+}
+
+impl CaseResult {
+    fn new() -> Self {
+        Self {
+            latency_us: vec![],
+            recalls: vec![],
+        }
+    }
+    fn median_latency_ms(&mut self) -> f64 {
+        self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        self.latency_us[self.latency_us.len() / 2] / 1000.0
+    }
+    fn mean_recall(&self) -> f64 {
+        self.recalls.iter().sum::<f64>() / self.recalls.len() as f64
+    }
+}
+
+fn main() {
+    println!("generating {TOTAL} x {DIMENSION} vectors...");
+    let data = generate_random_array_with_seed::<Float32Type>(TOTAL * DIMENSION, SEED);
+    let fsl = FixedSizeListArray::try_new_from_values(data, DIMENSION as i32).unwrap();
+    let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+
+    println!("building HNSW graph...");
+    let build_start = Instant::now();
+    let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap();
+    println!("built in {:.1}s", build_start.elapsed().as_secs_f32());
+
+    let mut rng = SmallRng::seed_from_u64(7);
+    let params = HnswQueryParams {
+        ef: EF,
+        lower_bound: None,
+        upper_bound: None,
+        dist_q_c: 0.0,
+        use_acorn: false,
+    };
+    let params_hi = HnswQueryParams {
+        ef: EF_HI,
+        ..params
+    };
+
+    // unfiltered control: this graph's recall ceiling at ef=EF
+    {
+        let mut control = CaseResult::new();
+        let all_mask = vec![true; TOTAL];
+        for _ in 0..QUERIES_PER_CASE {
+            let query_id = rng.random_range(0..TOTAL as u32);
+            let query = fsl.value(query_id as usize);
+            let truth = ground_truth(&storage, query.clone(), &all_mask);
+            let t = Instant::now();
+            let nodes = hnsw
+                .search_basic(query.clone(), K, &params, None, storage.as_ref())
+                .unwrap();
+            control.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+            let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+            control.recalls.push(recall(&got, &truth));
+        }
+        println!(
+            "\nunfiltered control (ef={EF}): {:.2} ms, recall@{K} {:.3}",
+            control.median_latency_ms(),
+            control.mean_recall()
+        );
+    }
+
+    println!(
+        "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>9} {:>7}",
+        "mask",
+        "sel%",
+        "basic ms",
+        "recall",
+        "basic4x ms",
+        "recall",
+        "acorn ms",
+        "recall",
+        "flat ms",
+        "recall"
+    );
+
+    for mask_kind in ["corr-in", "corr-out", "random"] {
+        for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] {
+            let mask_size = (TOTAL as f64 * selectivity) as usize;
+            let mut basic = CaseResult::new();
+            let mut basic_hi = CaseResult::new();
+            let mut acorn = CaseResult::new();
+            let mut flat = CaseResult::new();
+            let mut mask_generator = VisitedGenerator::new(TOTAL);
+
+            for _ in 0..QUERIES_PER_CASE {
+                let mut mask = vec![false; TOTAL];
+                let anchor_id = rng.random_range(0..TOTAL as u32);
+                let member_ids: Vec<u32> = match mask_kind {
+                    "random" => {
+                        let mut ids: Vec<u32> = (0..TOTAL as u32).collect();
+                        ids.shuffle(&mut rng);
+                        ids.truncate(mask_size);
+                        ids
+                    }
+                    _ => {
+                        // cluster: the nearest nodes to a random anchor
+                        let dists = all_distances(&storage, fsl.value(anchor_id as usize));
+                        let mut ids: Vec<u32> = (0..TOTAL as u32).collect();
+                        ids.sort_by(|&a, &b| {
+                            dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()
+                        });
+                        ids.truncate(mask_size);
+                        ids
+                    }
+                };
+                for &id in &member_ids {
+                    mask[id as usize] = true;
+                }
+
+                let query_id = match mask_kind {
+                    "corr-in" => *member_ids.choose(&mut rng).unwrap(),
+                    _ => rng.random_range(0..TOTAL as u32),
+                };
+                let query = fsl.value(query_id as usize);
+                let truth = ground_truth(&storage, query.clone(), &mask);
+
+                // search_basic (current traversal)
+                {
+                    let mut bitset = mask_generator.generate(TOTAL);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_basic(query.clone(), K, &params, Some(bitset), storage.as_ref())
+                        .unwrap();
+                    basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    basic.recalls.push(recall(&got, &truth));
+                }
+
+                // search_basic at 4x ef (recall-equalizing baseline)
+                {
+                    let mut bitset = mask_generator.generate(TOTAL);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_basic(query.clone(), K, &params_hi, Some(bitset), storage.as_ref())
+                        .unwrap();
+                    basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    basic_hi.recalls.push(recall(&got, &truth));
+                }
+
+                // search_acorn
+                {
+                    let mut bitset = mask_generator.generate(TOTAL);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_acorn(query.clone(), K, &params, &bitset, storage.as_ref())
+                        .unwrap();
+                    acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    acorn.recalls.push(recall(&got, &truth));
+                }
+
+                // flat scan over matching rows
+                {
+                    let t = Instant::now();
+                    let dist_calc = storage.dist_calculator(query.clone(), 0.0);
+                    let mut scored: Vec<(f32, u32)> = member_ids
+                        .iter()
+                        .map(|&id| (dist_calc.distance(id), id))
+                        .collect();
+                    scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+                    scored.truncate(K);
+                    flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = scored.iter().map(|&(_, id)| id).collect();
+                    flat.recalls.push(recall(&got, &truth));
+                }
+            }
+
+            println!(
+                "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>9.2} {:>7.3}",
+                mask_kind,
+                selectivity * 100.0,
+                basic.median_latency_ms(),
+                basic.mean_recall(),
+                basic_hi.median_latency_ms(),
+                basic_hi.mean_recall(),
+                acorn.median_latency_ms(),
+                acorn.mean_recall(),
+                flat.median_latency_ms(),
+                flat.mean_recall(),
+            );
+        }
+    }
+}
diff --git a/rust/lance-index/examples/acorn_bench_sift.rs b/rust/lance-index/examples/acorn_bench_sift.rs
new file mode 100644
index 00000000000..c781ae12577
--- /dev/null
+++ b/rust/lance-index/examples/acorn_bench_sift.rs
@@ -0,0 +1,329 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! SIFT1M variant of `acorn_bench`: compares `search_basic`, `search_acorn`,
+//! and a flat scan on the ANN_SIFT1M dataset (1M x 128, L2) with synthetic
+//! filter masks. The unfiltered control is checked against the official
+//! ground truth.
+//!
+//! Run: SIFT_DIR=/path/to/sift cargo run --release -p lance-index --example acorn_bench_sift
+//! Works with any texmex-format dataset, e.g. SIFT_DIR=/path/to/gist for GIST1M.
+
+#![allow(clippy::print_stdout)]
+
+use std::sync::Arc;
+use std::time::Instant;
+
+use arrow_array::{Array, FixedSizeListArray, Float32Array};
+use lance_arrow::FixedSizeListArrayExt;
+use lance_index::vector::flat::storage::FlatFloatStorage;
+use lance_index::vector::graph::VisitedGenerator;
+use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams};
+use lance_index::vector::storage::{DistCalculator, VectorStore};
+use lance_index::vector::v3::subindex::IvfSubIndex;
+use lance_linalg::distance::DistanceType;
+use rand::rngs::SmallRng;
+use rand::seq::SliceRandom;
+use rand::{Rng, SeedableRng};
+
+const K: usize = 10;
+const EF: usize = 100;
+const EF_HI: usize = 400;
+const QUERIES_PER_CASE: usize = 20;
+
+/// (flat values, dim, count)
+fn read_fvecs(path: &str) -> (Vec<f32>, usize, usize) {
+    let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}"));
+    let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize;
+    let record = 4 + dim * 4;
+    assert_eq!(bytes.len() % record, 0);
+    let count = bytes.len() / record;
+    let mut values = Vec::with_capacity(count * dim);
+    for row in 0..count {
+        let start = row * record + 4;
+        for i in 0..dim {
+            let offset = start + i * 4;
+            values.push(f32::from_le_bytes(
+                bytes[offset..offset + 4].try_into().unwrap(),
+            ));
+        }
+    }
+    (values, dim, count)
+}
+
+fn read_ivecs(path: &str) -> Vec<Vec<u32>> {
+    let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}"));
+    let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize;
+    let record = 4 + dim * 4;
+    assert_eq!(bytes.len() % record, 0);
+    (0..bytes.len() / record)
+        .map(|row| {
+            let start = row * record + 4;
+            (0..dim)
+                .map(|i| {
+                    let offset = start + i * 4;
+                    u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap())
+                })
+                .collect()
+        })
+        .collect()
+}
+
+fn all_distances(storage: &FlatFloatStorage, total: usize, query: Arc<dyn Array>) -> Vec<f32> {
+    let dist_calc = storage.dist_calculator(query, 0.0);
+    (0..total as u32).map(|i| dist_calc.distance(i)).collect()
+}
+
+fn ground_truth(
+    storage: &FlatFloatStorage,
+    total: usize,
+    query: Arc<dyn Array>,
+    mask: &[bool],
+) -> Vec<u32> {
+    let dists = all_distances(storage, total, query);
+    let mut ids: Vec<u32> = (0..total as u32).filter(|&i| mask[i as usize]).collect();
+    ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap());
+    ids.truncate(K);
+    ids
+}
+
+fn recall(got: &[u32], truth: &[u32]) -> f64 {
+    let hits = truth.iter().filter(|id| got.contains(id)).count();
+    hits as f64 / truth.len().max(1) as f64
+}
+
+struct CaseResult {
+    latency_us: Vec<f64>,
+    recalls: Vec<f64>,
+}
+
+impl CaseResult {
+    fn new() -> Self {
+        Self {
+            latency_us: vec![],
+            recalls: vec![],
+        }
+    }
+    fn median_latency_ms(&mut self) -> f64 {
+        self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        self.latency_us[self.latency_us.len() / 2] / 1000.0
+    }
+    fn mean_recall(&self) -> f64 {
+        self.recalls.iter().sum::<f64>() / self.recalls.len() as f64
+    }
+}
+
+fn main() {
+    let sift_dir = std::env::var("SIFT_DIR").expect("set SIFT_DIR to the extracted dataset dir");
+    let prefix = std::path::Path::new(&sift_dir)
+        .file_name()
+        .and_then(|name| name.to_str())
+        .expect("SIFT_DIR must end in the dataset name, e.g. sift or gist")
+        .to_string();
+    println!("loading {prefix} from {sift_dir}...");
+    let (base, dim, total) = read_fvecs(&format!("{sift_dir}/{prefix}_base.fvecs"));
+    let (queries, query_dim, num_queries) = read_fvecs(&format!("{sift_dir}/{prefix}_query.fvecs"));
+    let official_gt = read_ivecs(&format!("{sift_dir}/{prefix}_groundtruth.ivecs"));
+    assert_eq!(dim, query_dim);
+    println!("base {total} x {dim}, {num_queries} queries");
+
+    let fsl =
+        FixedSizeListArray::try_new_from_values(Float32Array::from(base), dim as i32).unwrap();
+    let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+    let query_vec = |i: usize| -> Arc<dyn Array> {
+        Arc::new(Float32Array::from(queries[i * dim..(i + 1) * dim].to_vec()))
+    };
+
+    println!("building HNSW graph...");
+    let build_start = Instant::now();
+    let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap();
+    println!("built in {:.1}s", build_start.elapsed().as_secs_f32());
+
+    let mut rng = SmallRng::seed_from_u64(7);
+    let params = HnswQueryParams {
+        ef: EF,
+        lower_bound: None,
+        upper_bound: None,
+        dist_q_c: 0.0,
+        use_acorn: false,
+    };
+    let params_hi = HnswQueryParams {
+        ef: EF_HI,
+        ..params
+    };
+
+    // unfiltered control against the official ground truth
+    {
+        let mut control = CaseResult::new();
+        for _ in 0..QUERIES_PER_CASE {
+            let query_id = rng.random_range(0..num_queries);
+            let query = query_vec(query_id);
+            let truth: Vec<u32> = official_gt[query_id][..K].to_vec();
+            let t = Instant::now();
+            let nodes = hnsw
+                .search_basic(query, K, &params, None, storage.as_ref())
+                .unwrap();
+            control.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+            let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+            control.recalls.push(recall(&got, &truth));
+        }
+        println!(
+            "\nunfiltered control (ef={EF}, official GT): {:.2} ms, recall@{K} {:.3}",
+            control.median_latency_ms(),
+            control.mean_recall()
+        );
+    }
+
+    println!(
+        "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7}",
+        "mask",
+        "sel%",
+        "basic ms",
+        "recall",
+        "basic4x ms",
+        "recall",
+        "acorn ms",
+        "recall",
+        "acorn4x ms",
+        "recall",
+        "flat ms",
+        "recall"
+    );
+
+    for mask_kind in ["corr-in", "corr-out", "random"] {
+        for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] {
+            let mask_size = (total as f64 * selectivity) as usize;
+            let mut basic = CaseResult::new();
+            let mut basic_hi = CaseResult::new();
+            let mut acorn = CaseResult::new();
+            let mut acorn_hi = CaseResult::new();
+            let mut flat = CaseResult::new();
+            let mut mask_generator = VisitedGenerator::new(total);
+
+            for _ in 0..QUERIES_PER_CASE {
+                let query_id = rng.random_range(0..num_queries);
+                let query = query_vec(query_id);
+
+                // corr-in: cluster around the query's own neighborhood
+                // corr-out: cluster around an unrelated base vector
+                let mut mask = vec![false; total];
+                let member_ids: Vec<u32> = match mask_kind {
+                    "random" => {
+                        let mut ids: Vec<u32> = (0..total as u32).collect();
+                        ids.shuffle(&mut rng);
+                        ids.truncate(mask_size);
+                        ids
+                    }
+                    _ => {
+                        let anchor: Arc<dyn Array> = match mask_kind {
+                            "corr-in" => query.clone(),
+                            _ => fsl.value(rng.random_range(0..total)),
+                        };
+                        let dists = all_distances(&storage, total, anchor);
+                        let mut ids: Vec<u32> = (0..total as u32).collect();
+                        ids.sort_by(|&a, &b| {
+                            dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()
+                        });
+                        ids.truncate(mask_size);
+                        ids
+                    }
+                };
+                for &id in &member_ids {
+                    mask[id as usize] = true;
+                }
+                let truth = ground_truth(&storage, total, query.clone(), &mask);
+
+                // search_basic (current traversal)
+                {
+                    let mut bitset = mask_generator.generate(total);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_basic(query.clone(), K, &params, Some(bitset), storage.as_ref())
+                        .unwrap();
+                    basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    basic.recalls.push(recall(&got, &truth));
+                }
+
+                // search_basic at 4x ef (recall-equalizing baseline)
+                {
+                    let mut bitset = mask_generator.generate(total);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_basic(query.clone(), K, &params_hi, Some(bitset), storage.as_ref())
+                        .unwrap();
+                    basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    basic_hi.recalls.push(recall(&got, &truth));
+                }
+
+                // search_acorn
+                {
+                    let mut bitset = mask_generator.generate(total);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_acorn(query.clone(), K, &params, &bitset, storage.as_ref())
+                        .unwrap();
+                    acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    acorn.recalls.push(recall(&got, &truth));
+                }
+
+                // search_acorn at 4x ef
+                {
+                    let mut bitset = mask_generator.generate(total);
+                    for &id in &member_ids {
+                        bitset.insert(id);
+                    }
+                    let t = Instant::now();
+                    let nodes = hnsw
+                        .search_acorn(query.clone(), K, &params_hi, &bitset, storage.as_ref())
+                        .unwrap();
+                    acorn_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = nodes.iter().map(|n| n.id).collect();
+                    acorn_hi.recalls.push(recall(&got, &truth));
+                }
+
+                // flat scan over matching rows
+                {
+                    let t = Instant::now();
+                    let dist_calc = storage.dist_calculator(query.clone(), 0.0);
+                    let mut scored: Vec<(f32, u32)> = member_ids
+                        .iter()
+                        .map(|&id| (dist_calc.distance(id), id))
+                        .collect();
+                    scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+                    scored.truncate(K);
+                    flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6);
+                    let got: Vec<u32> = scored.iter().map(|&(_, id)| id).collect();
+                    flat.recalls.push(recall(&got, &truth));
+                }
+            }
+
+            println!(
+                "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3}",
+                mask_kind,
+                selectivity * 100.0,
+                basic.median_latency_ms(),
+                basic.mean_recall(),
+                basic_hi.median_latency_ms(),
+                basic_hi.mean_recall(),
+                acorn.median_latency_ms(),
+                acorn.mean_recall(),
+                acorn_hi.median_latency_ms(),
+                acorn_hi.mean_recall(),
+                flat.median_latency_ms(),
+                flat.mean_recall(),
+            );
+        }
+    }
+}
diff --git a/rust/lance-index/protos-cache/cache.proto b/rust/lance-index/protos-cache/cache.proto
index b24a27055d7..a861e2ef652 100644
--- a/rust/lance-index/protos-cache/cache.proto
+++ b/rust/lance-index/protos-cache/cache.proto
@@ -28,6 +28,11 @@ message CompressedPostingHeader {
   PositionStorage position_storage = 4;
   // Only meaningful when position_storage == POSITION_STORAGE_SHARED.
   PositionStreamCodec position_stream_codec = 5;
+  // Number of documents in each compressed posting block. Older cache entries
+  // omit this field and decode as the legacy 128-doc block size.
+  uint32 block_size = 6;
+  // Whether an impact IPC section follows the posting/position sections.
+  bool has_impacts = 7;
 }
 
 // Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow
diff --git a/rust/lance-index/src/frag_reuse.rs b/rust/lance-index/src/frag_reuse.rs
index 4c70db44094..7072cdbe446 100644
--- a/rust/lance-index/src/frag_reuse.rs
+++ b/rust/lance-index/src/frag_reuse.rs
@@ -5,14 +5,16 @@
 //!
 //! The data structures and table-format logic live in
 //! [`lance_table::system_index::frag_reuse`]; this module re-exports them and
-//! implements the local [`Index`] trait for [`FragReuseIndex`].
+//! provides newtype wrappers that implement the [`Index`] and [`RowIdRemapper`]
+//! traits.
 
 use std::any::Any;
 use std::sync::Arc;
 
 use arrow_array::RecordBatch;
 use async_trait::async_trait;
-use lance_core::{Error, Result};
+use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use lance_select::RowAddrTreeMap;
 use roaring::{RoaringBitmap, RoaringTreemap};
 use serde::Serialize;
@@ -22,25 +24,22 @@ pub use lance_table::system_index::frag_reuse::*;
 use crate::scalar::RowIdRemapper;
 use crate::{Index, IndexType};
 
-impl RowIdRemapper for FragReuseIndex {
-    fn remap_row_id(&self, row_id: u64) -> Option<u64> {
-        self.remap_row_id(row_id)
-    }
+/// Newtype wrapping [`FragReuseIndex`] so that `lance-index` can implement
+/// the `Index` and `RowIdRemapper` traits (orphan rules prevent implementing
+/// them directly in `lance-table`).
+pub struct FragReuseIndexHandle(pub Arc<FragReuseIndex>);
 
-    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
-        self.remap_row_addrs_tree_map(row_addrs)
+impl std::fmt::Debug for FragReuseIndexHandle {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("FragReuseIndexHandle")
+            .field(&self.0)
+            .finish()
     }
+}
 
-    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
-        self.remap_row_ids_roaring_tree_map(row_ids)
-    }
-
-    fn remap_row_ids_record_batch(
-        &self,
-        batch: RecordBatch,
-        row_id_idx: usize,
-    ) -> Result<RecordBatch> {
-        self.remap_row_ids_record_batch(batch, row_id_idx)
+impl DeepSizeOf for FragReuseIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
     }
 }
 
@@ -50,7 +49,7 @@ struct FragReuseStatistics {
 }
 
 #[async_trait]
-impl Index for FragReuseIndex {
+impl Index for FragReuseIndexHandle {
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -61,10 +60,10 @@ impl Index for FragReuseIndex {
 
     fn statistics(&self) -> Result<serde_json::Value> {
         let stats = FragReuseStatistics {
-            num_versions: self.details.versions.len(),
+            num_versions: self.0.details.versions.len(),
         };
         serde_json::to_value(stats).map_err(|e| {
-            Error::internal(format!(
+            lance_core::Error::internal(format!(
                 "failed to serialize fragment reuse index statistics: {}",
                 e
             ))
@@ -83,3 +82,25 @@ impl Index for FragReuseIndex {
         unimplemented!()
     }
 }
+
+impl RowIdRemapper for FragReuseIndexHandle {
+    fn remap_row_id(&self, row_id: u64) -> Option<u64> {
+        self.0.remap_row_id(row_id)
+    }
+
+    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
+        self.0.remap_row_addrs_tree_map(row_addrs)
+    }
+
+    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
+        self.0.remap_row_ids_roaring_tree_map(row_ids)
+    }
+
+    fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch> {
+        self.0.remap_row_ids_record_batch(batch, row_id_idx)
+    }
+}
diff --git a/rust/lance-index/src/lib.rs b/rust/lance-index/src/lib.rs
index 61b45550367..c95e2ce609c 100644
--- a/rust/lance-index/src/lib.rs
+++ b/rust/lance-index/src/lib.rs
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
+#![cfg_attr(coverage, feature(coverage_attribute))]
 
 //! Lance secondary index library
 //!
@@ -9,16 +10,9 @@
 //! API stability is not guaranteed.
 //! </section>
 
-use std::{any::Any, sync::Arc};
-
 use crate::frag_reuse::FRAG_REUSE_INDEX_NAME;
 use crate::mem_wal::MEM_WAL_INDEX_NAME;
-use async_trait::async_trait;
-use lance_core::deepsize::DeepSizeOf;
-use lance_core::{Error, Result};
-use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
-use std::convert::TryFrom;
 
 pub mod frag_reuse;
 pub mod mem_wal;
@@ -33,6 +27,9 @@ pub mod vector;
 
 pub use crate::traits::*;
 
+// Re-export core traits from lance-index-core
+pub use lance_index_core::{Index, IndexParams, IndexType};
+
 pub const INDEX_FILE_NAME: &str = "index.idx";
 /// The name of the auxiliary index file.
 ///
@@ -75,280 +72,6 @@ pub mod cache_pb {
     include!(concat!(env!("OUT_DIR"), "/lance.index.cache.rs"));
 }
 
-/// Generic methods common across all types of secondary indices
-///
-#[async_trait]
-pub trait Index: Send + Sync + DeepSizeOf {
-    /// Cast to [Any].
-    fn as_any(&self) -> &dyn Any;
-
-    /// Cast to [Index]
-    fn as_index(self: Arc<Self>) -> Arc<dyn Index>;
-
-    /// Retrieve index statistics as a JSON Value
-    fn statistics(&self) -> Result<serde_json::Value>;
-
-    /// Prewarm the index.
-    ///
-    /// This will load the index into memory and cache it.
-    async fn prewarm(&self) -> Result<()>;
-
-    /// Get the type of the index
-    fn index_type(&self) -> IndexType;
-
-    /// Read through the index and determine which fragment ids are covered by the index
-    ///
-    /// This is a kind of slow operation.  It's better to use the fragment_bitmap.  This
-    /// only exists for cases where the fragment_bitmap has become corrupted or missing.
-    async fn calculate_included_frags(&self) -> Result<RoaringBitmap>;
-}
-
-/// Index Type
-#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf)]
-pub enum IndexType {
-    // Preserve 0-100 for simple indices.
-    Scalar = 0, // Legacy scalar index, alias to BTree
-
-    BTree = 1, // BTree
-
-    Bitmap = 2, // Bitmap
-
-    LabelList = 3, // LabelList
-
-    Inverted = 4, // Inverted
-
-    NGram = 5, // NGram
-
-    FragmentReuse = 6,
-
-    MemWal = 7,
-
-    ZoneMap = 8, // ZoneMap
-
-    BloomFilter = 9, // Bloom filter
-
-    RTree = 10, // RTree
-
-    Fm = 11, // FM-Index
-
-    // 100+ and up for vector index.
-    /// Flat vector index.
-    Vector = 100, // Legacy vector index, alias to IvfPq
-    IvfFlat = 101,
-    IvfSq = 102,
-    IvfPq = 103,
-    IvfHnswSq = 104,
-    IvfHnswPq = 105,
-    IvfHnswFlat = 106,
-    IvfRq = 107,
-}
-
-impl std::fmt::Display for IndexType {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            Self::Scalar | Self::BTree => write!(f, "BTree"),
-            Self::Bitmap => write!(f, "Bitmap"),
-            Self::LabelList => write!(f, "LabelList"),
-            Self::Inverted => write!(f, "Inverted"),
-            Self::NGram => write!(f, "NGram"),
-            Self::FragmentReuse => write!(f, "FragmentReuse"),
-            Self::MemWal => write!(f, "MemWal"),
-            Self::ZoneMap => write!(f, "ZoneMap"),
-            Self::BloomFilter => write!(f, "BloomFilter"),
-            Self::RTree => write!(f, "RTree"),
-            Self::Fm => write!(f, "Fm"),
-            Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"),
-            Self::IvfFlat => write!(f, "IVF_FLAT"),
-            Self::IvfSq => write!(f, "IVF_SQ"),
-            Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
-            Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
-            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
-            Self::IvfRq => write!(f, "IVF_RQ"),
-        }
-    }
-}
-
-impl TryFrom<i32> for IndexType {
-    type Error = Error;
-
-    fn try_from(value: i32) -> Result<Self> {
-        match value {
-            v if v == Self::Scalar as i32 => Ok(Self::Scalar),
-            v if v == Self::BTree as i32 => Ok(Self::BTree),
-            v if v == Self::Bitmap as i32 => Ok(Self::Bitmap),
-            v if v == Self::LabelList as i32 => Ok(Self::LabelList),
-            v if v == Self::NGram as i32 => Ok(Self::NGram),
-            v if v == Self::Inverted as i32 => Ok(Self::Inverted),
-            v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse),
-            v if v == Self::MemWal as i32 => Ok(Self::MemWal),
-            v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap),
-            v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter),
-            v if v == Self::RTree as i32 => Ok(Self::RTree),
-            v if v == Self::Fm as i32 => Ok(Self::Fm),
-            v if v == Self::Vector as i32 => Ok(Self::Vector),
-            v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat),
-            v if v == Self::IvfSq as i32 => Ok(Self::IvfSq),
-            v if v == Self::IvfPq as i32 => Ok(Self::IvfPq),
-            v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq),
-            v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq),
-            v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat),
-            v if v == Self::IvfRq as i32 => Ok(Self::IvfRq),
-            _ => Err(Error::invalid_input_source(
-                format!("the input value {} is not a valid IndexType", value).into(),
-            )),
-        }
-    }
-}
-
-impl TryFrom<&str> for IndexType {
-    type Error = Error;
-
-    fn try_from(value: &str) -> Result<Self> {
-        match value {
-            "BTree" | "BTREE" => Ok(Self::BTree),
-            "Bitmap" | "BITMAP" => Ok(Self::Bitmap),
-            "LabelList" | "LABELLIST" => Ok(Self::LabelList),
-            "Inverted" | "INVERTED" => Ok(Self::Inverted),
-            "NGram" | "NGRAM" => Ok(Self::NGram),
-            "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap),
-            "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter),
-            "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree),
-            "Fm" | "FM" => Ok(Self::Fm),
-            "Vector" | "VECTOR" => Ok(Self::Vector),
-            "IVF_FLAT" => Ok(Self::IvfFlat),
-            "IVF_SQ" => Ok(Self::IvfSq),
-            "IVF_PQ" => Ok(Self::IvfPq),
-            "IVF_RQ" => Ok(Self::IvfRq),
-            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
-            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
-            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
-            "FragmentReuse" => Ok(Self::FragmentReuse),
-            "MemWal" => Ok(Self::MemWal),
-            _ => Err(Error::invalid_input(format!(
-                "invalid index type: {}",
-                value
-            ))),
-        }
-    }
-}
-
-impl IndexType {
-    pub fn is_scalar(&self) -> bool {
-        matches!(
-            self,
-            Self::Scalar
-                | Self::BTree
-                | Self::Bitmap
-                | Self::LabelList
-                | Self::Inverted
-                | Self::NGram
-                | Self::ZoneMap
-                | Self::BloomFilter
-                | Self::RTree
-                | Self::Fm,
-        )
-    }
-
-    pub fn is_vector(&self) -> bool {
-        matches!(
-            self,
-            Self::Vector
-                | Self::IvfPq
-                | Self::IvfHnswSq
-                | Self::IvfHnswPq
-                | Self::IvfHnswFlat
-                | Self::IvfFlat
-                | Self::IvfSq
-                | Self::IvfRq
-        )
-    }
-
-    pub fn is_system(&self) -> bool {
-        matches!(self, Self::FragmentReuse | Self::MemWal)
-    }
-
-    /// Returns the current format version of the index type,
-    /// bump this when the index format changes.
-    /// Indices which higher version than these will be ignored for compatibility,
-    /// This would happen when creating index in a newer version of Lance,
-    /// but then opening the index in older version of Lance
-    pub fn version(&self) -> i32 {
-        match self {
-            Self::Scalar => 0,
-            Self::BTree => 0,
-            Self::Bitmap => 0,
-            Self::LabelList => 0,
-            Self::Inverted => 0,
-            Self::NGram => 0,
-            Self::FragmentReuse => 0,
-            Self::MemWal => 0,
-            Self::ZoneMap => 0,
-            Self::BloomFilter => 0,
-            Self::RTree => 0,
-            Self::Fm => 0,
-
-            // IMPORTANT: if any vector index subtype needs a format bump that is
-            // not backward compatible, its new version must be set to
-            // (current max vector index version + 1), even if only one subtype
-            // changed. Compatibility filtering currently cannot distinguish vector
-            // subtypes from details-only metadata, so vector versions effectively
-            // share one global monotonic compatibility level.
-            Self::Vector
-            | Self::IvfFlat
-            | Self::IvfSq
-            | Self::IvfPq
-            | Self::IvfHnswSq
-            | Self::IvfHnswPq
-            | Self::IvfHnswFlat => VECTOR_INDEX_VERSION as i32,
-            Self::IvfRq => IVF_RQ_INDEX_VERSION as i32,
-        }
-    }
-
-    /// Returns the target partition size for the index type.
-    ///
-    /// This is used to compute the number of partitions for the index.
-    /// The partition size is optimized for the best performance of the index.
-    ///
-    /// This is for vector indices only.
-    pub fn target_partition_size(&self) -> usize {
-        match self {
-            Self::Vector => 8192,
-            Self::IvfFlat => 4096,
-            Self::IvfSq => 8192,
-            Self::IvfPq => 8192,
-            Self::IvfRq => 4096,
-            Self::IvfHnswFlat => 1 << 20,
-            Self::IvfHnswSq => 1 << 20,
-            Self::IvfHnswPq => 1 << 20,
-            _ => 8192,
-        }
-    }
-
-    /// Returns the highest supported vector index version in this Lance build.
-    pub fn max_vector_version() -> u32 {
-        [
-            Self::Vector,
-            Self::IvfFlat,
-            Self::IvfSq,
-            Self::IvfPq,
-            Self::IvfHnswSq,
-            Self::IvfHnswPq,
-            Self::IvfHnswFlat,
-            Self::IvfRq,
-        ]
-        .into_iter()
-        .map(|index_type| index_type.version() as u32)
-        .max()
-        .unwrap_or(VECTOR_INDEX_VERSION)
-    }
-}
-
-pub trait IndexParams: Send + Sync {
-    fn as_any(&self) -> &dyn Any;
-
-    fn index_name(&self) -> &str;
-}
-
 #[derive(Serialize, Deserialize, Debug)]
 pub struct IndexMetadata {
     #[serde(rename = "type")]
diff --git a/rust/lance-index/src/mem_wal.rs b/rust/lance-index/src/mem_wal.rs
index 9bd72ff7866..879f4757919 100644
--- a/rust/lance-index/src/mem_wal.rs
+++ b/rust/lance-index/src/mem_wal.rs
@@ -5,13 +5,14 @@
 //!
 //! The data structures and table-format logic live in
 //! [`lance_table::system_index::mem_wal`]; this module re-exports them and
-//! implements the local [`Index`] trait for [`MemWalIndex`].
+//! provides a newtype wrapper that implements the [`Index`] trait.
 
 use std::any::Any;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use lance_core::Error;
+use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use roaring::RoaringBitmap;
 use serde::Serialize;
 
@@ -19,17 +20,28 @@ pub use lance_table::system_index::mem_wal::*;
 
 use crate::{Index, IndexType};
 
+/// Newtype wrapping [`MemWalIndex`] so that `lance-index` can implement
+/// the `Index` trait (orphan rules prevent implementing it directly in
+/// `lance-table`).
+pub struct MemWalIndexHandle(pub Arc<MemWalIndex>);
+
+impl DeepSizeOf for MemWalIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
+    }
+}
+
 #[derive(Serialize)]
 struct MemWalStatistics {
     num_shards: u32,
-    num_merged_generations: usize,
+    num_compacted_sstables: usize,
     num_shard_specs: usize,
     num_maintained_indexes: usize,
     num_index_catchup_entries: usize,
 }
 
 #[async_trait]
-impl Index for MemWalIndex {
+impl Index for MemWalIndexHandle {
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -38,23 +50,23 @@ impl Index for MemWalIndex {
         self
     }
 
-    fn statistics(&self) -> lance_core::Result<serde_json::Value> {
+    fn statistics(&self) -> Result<serde_json::Value> {
         let stats = MemWalStatistics {
-            num_shards: self.details.num_shards,
-            num_merged_generations: self.details.merged_generations.len(),
-            num_shard_specs: self.details.sharding_specs.len(),
-            num_maintained_indexes: self.details.maintained_indexes.len(),
-            num_index_catchup_entries: self.details.index_catchup.len(),
+            num_shards: self.0.details.num_shards,
+            num_compacted_sstables: self.0.details.compacted_sstables.len(),
+            num_shard_specs: self.0.details.sharding_specs.len(),
+            num_maintained_indexes: self.0.details.maintained_indexes.len(),
+            num_index_catchup_entries: self.0.details.index_catchup.len(),
         };
         serde_json::to_value(stats).map_err(|e| {
-            Error::internal(format!(
+            lance_core::Error::internal(format!(
                 "failed to serialize MemWAL index statistics: {}",
                 e
             ))
         })
     }
 
-    async fn prewarm(&self) -> lance_core::Result<()> {
+    async fn prewarm(&self) -> Result<()> {
         Ok(())
     }
 
@@ -62,7 +74,7 @@ impl Index for MemWalIndex {
         IndexType::MemWal
     }
 
-    async fn calculate_included_frags(&self) -> lance_core::Result<RoaringBitmap> {
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap> {
         Ok(RoaringBitmap::new())
     }
 }
diff --git a/rust/lance-index/src/metrics.rs b/rust/lance-index/src/metrics.rs
index 8c0c119a3c3..7d1ea2964c6 100644
--- a/rust/lance-index/src/metrics.rs
+++ b/rust/lance-index/src/metrics.rs
@@ -1,117 +1,4 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen";
-pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return";
-pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores";
-pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected";
-
-/// A trait used by the index to report metrics
-///
-/// Callers can implement this trait to collect metrics
-pub trait MetricsCollector: Send + Sync {
-    /// Record partition loads
-    ///
-    /// Many indices consist of partitions that may need to be loaded
-    /// into cache.  For example, an inverted index or ngram index has a
-    /// posting list for each token.
-    ///
-    /// In the ideal case, these shards are in the cache and will not need
-    /// to be loaded from disk.  This method should not be called if the
-    /// shard is in the cache.
-    fn record_parts_loaded(&self, num_parts: usize);
-
-    /// Record a shard load
-    fn record_part_load(&self) {
-        self.record_parts_loaded(1);
-    }
-
-    /// Record an index load
-    ///
-    /// This should be called when a scalar index is loaded from storage.
-    /// It should not be called if the index is already in memory.
-    fn record_index_loads(&self, num_indexes: usize);
-
-    /// Record an index load
-    fn record_index_load(&self) {
-        self.record_index_loads(1);
-    }
-
-    /// Record the number of "comparisons" made by the index
-    ///
-    /// What exactly constitutes a comparison depends on the index type.
-    /// For example, a B-tree index may make comparisons while searching for a value.
-    /// On the other hand, a bitmap index makes comparisons when computing the intersection
-    /// of two bitmaps.
-    ///
-    /// The goal is to provide some visibility into the compute cost of the search
-    fn record_comparisons(&self, num_comparisons: usize);
-
-    /// Record AND candidates returned from WAND alignment to the scoring loop.
-    ///
-    /// This excludes candidates pruned before `next()` returns. Use this with
-    /// `record_and_candidates_pruned_before_return` to recover total aligned
-    /// AND candidates.
-    fn record_and_candidates_seen(&self, _num_candidates: usize) {}
-
-    /// Record AND candidates pruned during WAND alignment before `next()` returns.
-    fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {}
-
-    fn record_and_full_scores(&self, _num_scores: usize) {}
-
-    fn record_freqs_collected(&self, _num_collections: usize) {}
-
-    /// Returns an optional sink for recording exact I/O statistics (bytes read,
-    /// IOPS, and requests) performed on behalf of this collector.
-    ///
-    /// Index implementations that read from a
-    /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to
-    /// their file readers so the I/O performed for a single query is measured
-    /// and attributed here.  The default returns `None`, meaning the caller does
-    /// not want I/O measured (and index implementations should then take their
-    /// normal, uninstrumented read path).
-    fn io_stats(&self) -> Option<lance_io::scheduler::IoStats> {
-        None
-    }
-}
-
-/// A no-op metrics collector that does nothing
-pub struct NoOpMetricsCollector;
-
-impl MetricsCollector for NoOpMetricsCollector {
-    fn record_parts_loaded(&self, _num_parts: usize) {}
-    fn record_index_loads(&self, _num_indexes: usize) {}
-    fn record_comparisons(&self, _num_comparisons: usize) {}
-}
-
-#[derive(Default)]
-pub struct LocalMetricsCollector {
-    pub parts_loaded: AtomicUsize,
-    pub index_loads: AtomicUsize,
-    pub comparisons: AtomicUsize,
-}
-
-impl LocalMetricsCollector {
-    pub fn dump_into(self, other: &dyn MetricsCollector) {
-        other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed));
-        other.record_index_loads(self.index_loads.load(Ordering::Relaxed));
-        other.record_comparisons(self.comparisons.load(Ordering::Relaxed));
-    }
-}
-
-impl MetricsCollector for LocalMetricsCollector {
-    fn record_parts_loaded(&self, num_parts: usize) {
-        self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed);
-    }
-
-    fn record_index_loads(&self, num_indexes: usize) {
-        self.index_loads.fetch_add(num_indexes, Ordering::Relaxed);
-    }
-
-    fn record_comparisons(&self, num_comparisons: usize) {
-        self.comparisons
-            .fetch_add(num_comparisons, Ordering::Relaxed);
-    }
-}
+pub use lance_index_core::metrics::*;
diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs
index f7b158ed558..92549e289d8 100644
--- a/rust/lance-index/src/scalar.rs
+++ b/rust/lance-index/src/scalar.rs
@@ -4,19 +4,13 @@
 //! Scalar indices for metadata search & filtering
 
 use arrow::buffer::{OffsetBuffer, ScalarBuffer};
-use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array};
-use arrow_schema::{Field, Schema};
-use async_trait::async_trait;
-use bytes::Bytes;
+use arrow_array::ListArray;
+use arrow_schema::Field;
 use datafusion::functions::regex::regexplike::RegexpLikeFunc;
 use datafusion::functions::string::contains::ContainsFunc;
 use datafusion::functions_nested::array_has;
-use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion_common::{Column, scalar::ScalarValue};
-use lance_core::utils::row_addr_remap::RowAddrRemap;
-use std::collections::{HashMap, HashSet};
-use std::fmt::Debug;
-use std::pin::Pin;
+use std::collections::HashSet;
 use std::{any::Any, ops::Bound, sync::Arc};
 
 use datafusion_expr::{
@@ -24,17 +18,17 @@ use datafusion_expr::{
     expr::{Like, ScalarFunction},
 };
 use inverted::query::{FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, fill_fts_query_column};
-use lance_core::deepsize::DeepSizeOf;
-use lance_core::{Error, Result};
-use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter};
-use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
-use roaring::{RoaringBitmap, RoaringTreemap};
-use serde::Serialize;
-
-use crate::metrics::MetricsCollector;
-use crate::scalar::registry::TrainingCriteria;
-use crate::{Index, IndexParams, IndexType};
-pub use lance_table::format::IndexFile;
+use lance_core::Result;
+
+use lance_datafusion::udf::CONTAINS_TOKENS_UDF;
+
+use crate::IndexParams;
+pub use crate::metrics::MetricsCollector;
+pub use lance_index_core::scalar::{
+    AnyQuery, BuiltinIndexType, CreatedIndex, IndexFile, IndexReader, IndexStore, IndexWriter,
+    LANCE_SCALAR_INDEX, OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams,
+    SearchResult, TrainingCriteria, TrainingOrdering, UpdateCriteria,
+};
 
 pub mod bitmap;
 pub mod bloomfilter;
@@ -49,121 +43,44 @@ pub mod ngram;
 pub mod registry;
 #[cfg(feature = "geo")]
 pub mod rtree;
+pub mod seed;
 pub mod zoned;
 pub mod zonemap;
 
 pub use inverted::tokenizer::InvertedIndexParams;
-use lance_datafusion::udf::CONTAINS_TOKENS_UDF;
 
-pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index";
-
-/// Builtin index types supported by the Lance library
+/// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a
+/// `Vec<`[`lance_table::format::IndexFile`]`>`.
 ///
-/// This is primarily for convenience to avoid a bunch of string
-/// constants and provide some auto-complete.  This type should not
-/// be used in the manifest as plugins cannot add new entries.
-#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
-pub enum BuiltinIndexType {
-    BTree,
-    Bitmap,
-    LabelList,
-    NGram,
-    ZoneMap,
-    BloomFilter,
-    RTree,
-    Inverted,
-    Fm,
-}
-
-impl BuiltinIndexType {
-    pub fn as_str(&self) -> &str {
-        match self {
-            Self::BTree => "btree",
-            Self::Bitmap => "bitmap",
-            Self::LabelList => "labellist",
-            Self::NGram => "ngram",
-            Self::ZoneMap => "zonemap",
-            Self::Inverted => "inverted",
-            Self::BloomFilter => "bloomfilter",
-            Self::RTree => "rtree",
-            Self::Fm => "fm",
-        }
-    }
-}
-
-impl TryFrom<IndexType> for BuiltinIndexType {
-    type Error = Error;
-
-    fn try_from(value: IndexType) -> Result<Self> {
-        match value {
-            IndexType::BTree => Ok(Self::BTree),
-            IndexType::Bitmap => Ok(Self::Bitmap),
-            IndexType::LabelList => Ok(Self::LabelList),
-            IndexType::NGram => Ok(Self::NGram),
-            IndexType::ZoneMap => Ok(Self::ZoneMap),
-            IndexType::Inverted => Ok(Self::Inverted),
-            IndexType::BloomFilter => Ok(Self::BloomFilter),
-            IndexType::RTree => Ok(Self::RTree),
-            IndexType::Fm => Ok(Self::Fm),
-            _ => Err(Error::index("Invalid index type".to_string())),
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct ScalarIndexParams {
-    /// The type of index to create
-    ///
-    /// Plugins may add additional index types.  Index type lookup is case-insensitive.
-    pub index_type: String,
-    /// The parameters to train the index
-    ///
-    /// This should be a JSON string.  The contents of the JSON string will be specific to the
-    /// index type.  If not set, then default parameters will be used for the index type.
-    pub params: Option<String>,
-}
-
-impl Default for ScalarIndexParams {
-    fn default() -> Self {
-        Self {
-            index_type: BuiltinIndexType::BTree.as_str().to_string(),
-            params: None,
-        }
-    }
-}
-
-impl ScalarIndexParams {
-    /// Creates a new ScalarIndexParams from one of the builtin index types
-    pub fn for_builtin(index_type: BuiltinIndexType) -> Self {
-        Self {
-            index_type: index_type.as_str().to_string(),
-            params: None,
-        }
-    }
-
-    /// Create a new ScalarIndexParams with the given index type
-    pub fn new(index_type: String) -> Self {
-        Self {
-            index_type,
-            params: None,
-        }
-    }
-
-    /// Set the parameters for the index
-    pub fn with_params<ParamsType: Serialize>(mut self, params: &ParamsType) -> Self {
-        self.params = Some(serde_json::to_string(params).unwrap());
-        self
-    }
-}
-
-impl IndexParams for ScalarIndexParams {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
-    fn index_name(&self) -> &str {
-        LANCE_SCALAR_INDEX
-    }
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn index_files_to_table(
+    files: Vec<lance_index_core::scalar::IndexFile>,
+) -> Vec<lance_table::format::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_table::format::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
+}
+
+/// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a
+/// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`.
+///
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn table_files_to_index(
+    files: Vec<lance_table::format::IndexFile>,
+) -> Vec<lance_index_core::scalar::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_index_core::scalar::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
 }
 
 impl IndexParams for InvertedIndexParams {
@@ -176,180 +93,6 @@ impl IndexParams for InvertedIndexParams {
     }
 }
 
-/// Trait for storing an index (or parts of an index) into storage
-#[async_trait]
-pub trait IndexWriter: Send {
-    /// Writes a record batch into the file, returning the 0-based index of the batch in the file
-    ///
-    /// E.g. if this is the third time this is called this method will return 2
-    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64>;
-    /// Adds a global buffer and returns its index.
-    async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
-        Err(Error::not_supported(
-            "global buffers are not supported by this index writer",
-        ))
-    }
-    /// Finishes writing the file and closes the file
-    async fn finish(&mut self) -> Result<IndexFile>;
-    /// Finishes writing the file and closes the file with additional metadata
-    async fn finish_with_metadata(
-        &mut self,
-        metadata: HashMap<String, String>,
-    ) -> Result<IndexFile>;
-}
-
-/// Trait for reading an index (or parts of an index) from storage
-#[async_trait]
-pub trait IndexReader: Send + Sync {
-    /// Read the n-th record batch from the file
-    async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch>;
-    /// Reads a global buffer by index.
-    async fn read_global_buffer(&self, _index: u32) -> Result<Bytes> {
-        Err(Error::not_supported(
-            "global buffers are not supported by this index reader",
-        ))
-    }
-    /// Read the range of rows from the file.
-    /// If projection is Some, only return the columns in the projection,
-    /// nested columns like Some(&["x.y"]) are not supported.
-    /// If projection is None, return all columns.
-    async fn read_range(
-        &self,
-        range: std::ops::Range<usize>,
-        projection: Option<&[&str]>,
-    ) -> Result<RecordBatch>;
-    /// Read multiple ranges and concatenate into a single batch.
-    /// Default impl runs `read_range`s in parallel via `try_join_all`.
-    async fn read_ranges(
-        &self,
-        ranges: &[std::ops::Range<usize>],
-        projection: Option<&[&str]>,
-    ) -> Result<RecordBatch> {
-        if ranges.is_empty() {
-            return self.read_range(0..0, projection).await;
-        }
-        let futures = ranges
-            .iter()
-            .map(|r| self.read_range(r.clone(), projection));
-        let batches = futures::future::try_join_all(futures).await?;
-        let schema = batches[0].schema();
-        Ok(arrow_select::concat::concat_batches(&schema, &batches)?)
-    }
-    /// Read a range of rows as a stream of record batches.
-    ///
-    /// This allows the caller to process rows incrementally without loading the
-    /// entire range into memory at once.
-    ///
-    /// The default implementation falls back to [`Self::read_range`] and wraps
-    /// the result in a single-item stream.
-    async fn read_range_stream(
-        &self,
-        range: std::ops::Range<usize>,
-        projection: Option<&[&str]>,
-    ) -> Result<Pin<Box<dyn RecordBatchStream>>> {
-        let batch = self.read_range(range, projection).await?;
-        let schema = batch.schema();
-        Ok(Box::pin(RecordBatchStreamAdapter::new(
-            schema,
-            futures::stream::once(async move { Ok(batch) }),
-        )))
-    }
-    /// Return the number of batches in the file
-    async fn num_batches(&self, batch_size: u64) -> u32;
-    /// Return the number of rows in the file
-    fn num_rows(&self) -> usize;
-    /// Return the metadata of the file
-    fn schema(&self) -> &lance_core::datatypes::Schema;
-    /// Best-effort on-disk byte size of the file when the reader already knows it
-    /// without extra I/O, else `None`. Used to size prewarm chunks.
-    fn file_size_bytes(&self) -> Option<u64> {
-        None
-    }
-}
-
-/// Trait abstracting I/O away from index logic
-///
-/// Scalar indices are currently serialized as indexable arrow record batches stored in
-/// named "files".  The index store is responsible for serializing and deserializing
-/// these batches into file data (e.g. as .lance files or .parquet files, etc.)
-#[async_trait]
-pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
-    fn as_any(&self) -> &dyn Any;
-    fn clone_arc(&self) -> Arc<dyn IndexStore>;
-
-    /// Suggested I/O parallelism for the store
-    fn io_parallelism(&self) -> usize;
-
-    /// Create a new file and return a writer to store data in the file
-    async fn new_index_file(&self, name: &str, schema: Arc<Schema>)
-    -> Result<Box<dyn IndexWriter>>;
-
-    /// Open an existing file for retrieval
-    async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>>;
-
-    /// Return a store that submits its I/O at the given base priority.
-    fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore>;
-
-    /// Copy a range of batches from an index file from this store to another
-    ///
-    /// This is often useful when remapping or updating
-    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<IndexFile>;
-
-    /// Copy an index file from this store to a new name in another store, leaving the source intact
-    async fn copy_index_file_to(
-        &self,
-        name: &str,
-        new_name: &str,
-        dest_store: &dyn IndexStore,
-    ) -> Result<IndexFile> {
-        if name == new_name {
-            self.copy_index_file(name, dest_store).await
-        } else {
-            Err(Error::not_supported(format!(
-                "copying index file {name} to {new_name} is not supported by this index store"
-            )))
-        }
-    }
-
-    /// Rename an index file
-    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile>;
-
-    /// Delete an index file (used in the tmp spill store to keep tmp size down)
-    async fn delete_index_file(&self, name: &str) -> Result<()>;
-
-    /// List all files in the index directory with their sizes.
-    ///
-    /// Returns a list of (relative_path, size_bytes) tuples.
-    /// Used to capture file metadata after index creation/modification.
-    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>>;
-}
-
-/// Different scalar indices may support different kinds of queries
-///
-/// For example, a btree index can support a wide range of queries (e.g. x > 7)
-/// while an index based on FTS only supports queries like "x LIKE 'foo'"
-///
-/// This trait is used when we need an object that can represent any kind of query
-///
-/// Note: if you are implementing this trait for a query type then you probably also
-/// need to implement the [crate::scalar::expression::ScalarQueryParser] trait to
-/// create instances of your query at parse time.
-pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync {
-    /// Cast the query as Any to allow for downcasting
-    fn as_any(&self) -> &dyn Any;
-    /// Format the query as a string for display purposes
-    fn format(&self, col: &str) -> String;
-    /// Convert the query to a datafusion expression
-    fn to_expr(&self, col: String) -> Expr;
-    /// Compare this query to another query
-    fn dyn_eq(&self, other: &dyn AnyQuery) -> bool;
-}
-
-impl PartialEq for dyn AnyQuery {
-    fn eq(&self, other: &Self) -> bool {
-        self.dyn_eq(other)
-    }
-}
 /// A full text search query
 #[derive(Debug, Clone, PartialEq)]
 pub struct FullTextSearchQuery {
@@ -892,149 +635,6 @@ impl AnyQuery for GeoQuery {
     }
 }
 
-/// The result of a search operation against a scalar index
-#[derive(Debug, PartialEq)]
-pub enum SearchResult {
-    /// The exact row ids that satisfy the query
-    Exact(NullableRowAddrSet),
-    /// Any row id satisfying the query will be in this set but not every
-    /// row id in this set will satisfy the query, a further recheck step
-    /// is needed
-    AtMost(NullableRowAddrSet),
-    /// All of the given row ids satisfy the query but there may be more
-    ///
-    /// No scalar index actually returns this today but it can arise from
-    /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x)))
-    AtLeast(NullableRowAddrSet),
-}
-
-impl SearchResult {
-    pub fn exact(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn at_most(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn at_least(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn with_nulls(self, nulls: impl Into<RowAddrTreeMap>) -> Self {
-        match self {
-            Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())),
-            Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())),
-            Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())),
-        }
-    }
-
-    pub fn row_addrs(&self) -> &NullableRowAddrSet {
-        match self {
-            Self::Exact(row_addrs) => row_addrs,
-            Self::AtMost(row_addrs) => row_addrs,
-            Self::AtLeast(row_addrs) => row_addrs,
-        }
-    }
-
-    pub fn is_exact(&self) -> bool {
-        matches!(self, Self::Exact(_))
-    }
-}
-
-/// Brief information about an index that was created
-pub struct CreatedIndex {
-    /// The details of the index that was created
-    ///
-    /// These should be stored somewhere as they will be needed to
-    /// load the index later.
-    pub index_details: prost_types::Any,
-    /// The version of the index that was created
-    ///
-    /// This can be used to determine if a reader is able to load the index.
-    pub index_version: u32,
-    /// List of files and their sizes for this index
-    ///
-    /// This enables skipping HEAD calls when opening indices and provides
-    /// visibility into index storage size via describe_indices().
-    pub files: Vec<IndexFile>,
-}
-
-/// The criteria that specifies how to update an index
-pub struct UpdateCriteria {
-    /// If true, then we need to read the old data to update the index
-    ///
-    /// This should be avoided if possible but is left in for some legacy paths
-    pub requires_old_data: bool,
-    /// The criteria required for data (both old and new)
-    pub data_criteria: TrainingCriteria,
-}
-
-/// Filter used when merging existing scalar-index rows during update.
-///
-/// The caller must pick a filter mode that matches the row-id semantics of the
-/// dataset:
-/// - address-style row IDs: fragment filtering is valid
-/// - stable row IDs: use exact row-id membership instead
-#[derive(Debug, Clone)]
-pub enum OldIndexDataFilter {
-    /// Keeps track of which fragments are still valid and which are no longer valid.
-    ///
-    /// This is valid for address-style row IDs.
-    Fragments {
-        to_keep: RoaringBitmap,
-        to_remove: RoaringBitmap,
-    },
-    /// Keep old rows whose row IDs are in this exact allow-list.
-    ///
-    /// This is required for stable row IDs, where row IDs are opaque and
-    /// should not be interpreted as encoded row addresses.
-    RowIds(RowAddrTreeMap),
-}
-
-impl OldIndexDataFilter {
-    /// Build a boolean mask that keeps only row IDs selected by this filter.
-    pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray {
-        match self {
-            Self::Fragments { to_keep, .. } => row_ids
-                .iter()
-                .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32)))
-                .collect(),
-            Self::RowIds(valid_row_ids) => row_ids
-                .iter()
-                .map(|id| id.map(|id| valid_row_ids.contains(id)))
-                .collect(),
-        }
-    }
-
-    /// Apply this filter in place to a set of existing (old) row ids/addresses,
-    /// retaining only the rows the filter selects to keep. Used by index types
-    /// that merge old postings directly (e.g. bitmap) instead of re-scanning a
-    /// row-id array through [`Self::filter_row_ids`].
-    pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) {
-        match self {
-            Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()),
-            Self::RowIds(valid_row_ids) => *rows &= valid_row_ids,
-        }
-    }
-}
-
-impl UpdateCriteria {
-    pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self {
-        Self {
-            requires_old_data: true,
-            data_criteria,
-        }
-    }
-
-    pub fn only_new_data(data_criteria: TrainingCriteria) -> Self {
-        Self {
-            requires_old_data: false,
-            data_criteria,
-        }
-    }
-}
-
 /// Compute the lexicographically next prefix by incrementing the last character's code point.
 /// Returns None if no valid upper bound exists.
 ///
@@ -1091,77 +691,6 @@ fn next_unicode_char(c: char) -> Option<char> {
     char::from_u32(next_cp)
 }
 
-/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries
-#[async_trait]
-pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
-    /// Search the scalar index
-    ///
-    /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered
-    async fn search(
-        &self,
-        query: &dyn AnyQuery,
-        metrics: &dyn MetricsCollector,
-    ) -> Result<SearchResult>;
-
-    /// Returns true if the remap operation is supported
-    fn can_remap(&self) -> bool;
-
-    /// Remap the row ids, creating a new remapped version of this index in `dest_store`
-    async fn remap(
-        &self,
-        mapping: &RowAddrRemap,
-        dest_store: &dyn IndexStore,
-    ) -> Result<CreatedIndex>;
-
-    /// Add the new data into the index, creating an updated version of the index in `dest_store`
-    ///
-    /// If `old_data_filter` is provided, old index data will be filtered before
-    /// merge according to the chosen filter mode.
-    async fn update(
-        &self,
-        new_data: SendableRecordBatchStream,
-        dest_store: &dyn IndexStore,
-        old_data_filter: Option<OldIndexDataFilter>,
-    ) -> Result<CreatedIndex>;
-
-    /// Returns the criteria that will be used to update the index
-    fn update_criteria(&self) -> UpdateCriteria;
-
-    /// Derive the index parameters from the current index
-    ///
-    /// This returns a ScalarIndexParams that can be used to recreate an index
-    /// with the same configuration on another dataset.
-    fn derive_index_params(&self) -> Result<ScalarIndexParams>;
-
-    /// Global `[min, max]` of the indexed column from index metadata, without a
-    /// scan, or `None` if this index type cannot supply a sound bound. When
-    /// `Some`, the range is a superset of live values (conservative under
-    /// deletes): safe to prune with, not guaranteed tight.
-    fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> {
-        None
-    }
-}
-
-/// Abstraction over any type that can remap row IDs during index loading.
-///
-/// This decouples scalar index plugins from the table-level [`crate::frag_reuse::FragReuseIndex`]
-/// type.  [`crate::frag_reuse::FragReuseIndex`] implements this trait, but callers may also
-/// supply custom implementations for testing or other remapping strategies.
-pub trait RowIdRemapper: Send + Sync + std::fmt::Debug {
-    /// Remap a single row id.  Returns `None` if the row was deleted.
-    fn remap_row_id(&self, row_id: u64) -> Option<u64>;
-    /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows.
-    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap;
-    /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows.
-    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap;
-    /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows.
-    fn remap_row_ids_record_batch(
-        &self,
-        batch: RecordBatch,
-        row_id_idx: usize,
-    ) -> Result<RecordBatch>;
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/rust/lance-index/src/scalar/bitmap.rs b/rust/lance-index/src/scalar/bitmap.rs
index 98a07ff827d..91595b277f0 100644
--- a/rust/lance-index/src/scalar/bitmap.rs
+++ b/rust/lance-index/src/scalar/bitmap.rs
@@ -891,8 +891,7 @@ impl BitmapBatchWriter {
             return Ok(());
         }
         let keys_array =
-            ScalarValue::iter_to_array(self.keys.drain(..).collect::<Vec<_>>().into_iter())
-                .unwrap();
+            ScalarValue::iter_to_array(self.keys.drain(..).collect::<Vec<_>>()).unwrap();
         let total_size: usize = self.serialized.iter().map(|b| b.len()).sum();
         let mut binary_builder = BinaryBuilder::with_capacity(self.serialized.len(), total_size);
         for b in self.serialized.drain(..) {
@@ -1123,10 +1122,7 @@ async fn drain_same_key_bitmaps(
     let merged_key = OrderableScalarValue(key);
     advance_cursor_and_push(cursors, heap, item.shard_idx).await?;
 
-    loop {
-        let Some(Reverse(next_item)) = heap.peek() else {
-            break;
-        };
+    while let Some(Reverse(next_item)) = heap.peek() {
         if next_item.key != merged_key {
             break;
         }
@@ -1280,7 +1276,7 @@ impl BitmapIndexPlugin {
             let bitmap_size = bytes.len();
 
             if cur_bytes + bitmap_size > MAX_BITMAP_ARRAY_LENGTH {
-                let keys_array = ScalarValue::iter_to_array(cur_keys.clone().into_iter()).unwrap();
+                let keys_array = ScalarValue::iter_to_array(cur_keys.clone()).unwrap();
                 let mut binary_builder = BinaryBuilder::new();
                 for b in &cur_bitmaps {
                     binary_builder.append_value(b);
diff --git a/rust/lance-index/src/scalar/bloomfilter.rs b/rust/lance-index/src/scalar/bloomfilter.rs
index 41bcb5a8b11..9360fd943a1 100644
--- a/rust/lance-index/src/scalar/bloomfilter.rs
+++ b/rust/lance-index/src/scalar/bloomfilter.rs
@@ -7,6 +7,7 @@
 //! It is a space-efficient data structure that can be used to test whether an element is a member of a set.
 //! It's an inexact filter - they may include false positives that require rechecking.
 
+use crate::pb;
 use crate::scalar::expression::{BloomFilterQueryParser, ScalarQueryParser};
 use crate::scalar::registry::{
     BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
@@ -14,15 +15,17 @@ use crate::scalar::registry::{
 use crate::scalar::{
     BloomFilterQuery, BuiltinIndexType, CreatedIndex, IndexFile, ScalarIndexParams, UpdateCriteria,
 };
-use crate::{Any, pb};
 use arrow_array::{Array, UInt64Array};
 use arrow_schema::{DataType, Field};
 use lance_arrow_stats::StatisticsAccumulator;
 use lance_core::utils::bloomfilter::as_bytes;
 use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder};
 use lance_core::utils::row_addr_remap::RowAddrRemap;
+use lance_select::RowAddrTreeMap;
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 
+use std::collections::HashMap;
 use std::sync::LazyLock;
 
 use datafusion::execution::SendableRecordBatchStream;
@@ -44,8 +47,10 @@ use super::zoned::{ZoneBound, ZoneProcessor, ZoneTrainer, rebuild_zones, search_
 
 const BLOOMFILTER_FILENAME: &str = "bloomfilter.lance";
 const BLOOMFILTER_ITEM_META_KEY: &str = "bloomfilter_item";
+const NULL_BITMAP_META_KEY: &str = "null_bitmap";
 const BLOOMFILTER_PROBABILITY_META_KEY: &str = "bloomfilter_probability";
 const BLOOMFILTER_INDEX_VERSION: u32 = 0;
+pub const MAX_BINARY_VALUE_BUFFER_LEN: u64 = i32::MAX as u64;
 
 #[derive(Debug, Clone)]
 struct BloomFilterStatistics {
@@ -80,18 +85,21 @@ pub struct BloomFilterIndex {
     number_of_items: u64,
     // Probability of false positives, fraction between 0 and 1
     probability: f64,
+    // Exact set of null row addresses; None for older indices without this bitmap.
+    null_rows: Option<RowAddrTreeMap>,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
 }
 
 impl DeepSizeOf for BloomFilterIndex {
     fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
-        self.zones.deep_size_of_children(context)
+        self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context)
     }
 }
 
 impl BloomFilterIndex {
     async fn load(
         store: Arc<dyn IndexStore>,
-        _fri: Option<Arc<dyn RowIdRemapper>>,
+        fri: Option<Arc<dyn RowIdRemapper>>,
         _index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let index_file = store.open_index_file(BLOOMFILTER_FILENAME).await?;
@@ -112,10 +120,22 @@ impl BloomFilterIndex {
             .and_then(|bs| bs.parse().ok())
             .unwrap_or(*DEFAULT_PROBABILITY);
 
+        let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) {
+            let idx = idx_str.parse::<u32>().map_err(|e| {
+                Error::invalid_input(format!("invalid null bitmap buffer index: {e}"))
+            })?;
+            let bytes = index_file.read_global_buffer(idx).await?;
+            Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?)
+        } else {
+            None
+        };
+
         Ok(Arc::new(Self::try_from_serialized(
             bloom_data,
             number_of_items,
             probability,
+            null_rows,
+            fri,
         )?))
     }
 
@@ -123,13 +143,16 @@ impl BloomFilterIndex {
         data: RecordBatch,
         number_of_items: u64,
         probability: f64,
+        null_rows: Option<RowAddrTreeMap>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Self> {
         if data.num_rows() == 0 {
-            // Return empty index for empty data
             return Ok(Self {
                 zones: Vec::new(),
                 number_of_items,
                 probability,
+                null_rows,
+                frag_reuse_index,
             });
         }
 
@@ -210,6 +233,8 @@ impl BloomFilterIndex {
             zones: blocks,
             number_of_items,
             probability,
+            null_rows,
+            frag_reuse_index,
         })
     }
 
@@ -415,11 +440,21 @@ impl ScalarIndex for BloomFilterIndex {
         metrics: &dyn MetricsCollector,
     ) -> Result<SearchResult> {
         let query = query.as_any().downcast_ref::<BloomFilterQuery>().unwrap();
+        if let BloomFilterQuery::IsNull() = query
+            && let Some(null_rows) = &self.null_rows
+        {
+            return Ok(SearchResult::exact(null_rows.clone()));
+        }
+
         search_zones(&self.zones, metrics, |block| {
             self.evaluate_block_against_query(block, query)
         })
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        true
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
@@ -448,18 +483,28 @@ impl ScalarIndex for BloomFilterIndex {
 
         let processor = BloomFilterProcessor::new(params.clone())?;
         let trainer = ZoneTrainer::new(processor, params.number_of_items)?;
-        let updated_blocks = rebuild_zones(&self.zones, trainer, new_data).await?;
+        let (updated_blocks, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?;
+
+        // Merge existing and new null rows.  If the existing index had no null bitmap
+        // (legacy format — null positions unknown), preserve that None: updating cannot
+        // recover the missing information, and claiming the result has zero nulls would
+        // be a false negative.  Only a full retrain produces a fresh, complete bitmap.
+        let merged_null_rows = self.null_rows.as_ref().map(|existing| {
+            let mut merged = existing.clone();
+            merged |= &new_null_rows;
+            merged
+        });
 
         // Write the combined zones back to storage
         let mut builder = BloomFilterIndexBuilder::try_new(params)?;
         builder.blocks = updated_blocks;
-        let file = builder.write_index(dest_store).await?;
+        builder.null_rows = merged_null_rows;
+        let files = builder.write_index(dest_store).await?;
 
         Ok(CreatedIndex {
-            index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
-                .unwrap(),
+            index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?,
             index_version: BLOOMFILTER_INDEX_VERSION,
-            files: vec![file],
+            files,
         })
     }
 
@@ -478,6 +523,144 @@ impl ScalarIndex for BloomFilterIndex {
     }
 }
 
+fn remap_zone(
+    zone: &BloomFilterStatistics,
+    remapper: &dyn RowIdRemapper,
+) -> Vec<BloomFilterStatistics> {
+    let zone_start = (zone.bound.fragment_id << 32).saturating_add(zone.bound.start);
+    let mut remapped = (0..zone.bound.length as u64)
+        .filter_map(|offset| remapper.remap_row_id(zone_start.saturating_add(offset)))
+        .collect::<Vec<_>>();
+    remapped.sort_unstable();
+    remapped.dedup();
+
+    let mut zones = Vec::new();
+    let mut run_start = None;
+    let mut previous = 0u64;
+    for row_id in remapped {
+        if run_start.is_none() {
+            run_start = Some(row_id);
+        } else if row_id != previous.saturating_add(1) || row_id >> 32 != previous >> 32 {
+            let start = run_start.take().unwrap();
+            zones.push(BloomFilterStatistics {
+                bound: ZoneBound {
+                    fragment_id: start >> 32,
+                    start: start & u64::from(u32::MAX),
+                    length: (previous - start + 1) as usize,
+                },
+                has_null: zone.has_null,
+                bloom_filter: zone.bloom_filter.clone(),
+            });
+            run_start = Some(row_id);
+        }
+        previous = row_id;
+    }
+    if let Some(start) = run_start {
+        zones.push(BloomFilterStatistics {
+            bound: ZoneBound {
+                fragment_id: start >> 32,
+                start: start & u64::from(u32::MAX),
+                length: (previous - start + 1) as usize,
+            },
+            has_null: zone.has_null,
+            bloom_filter: zone.bloom_filter.clone(),
+        });
+    }
+    zones
+}
+
+fn checked_merged_payload_size(current: u64, additional: usize) -> Result<u64> {
+    let total = current
+        .checked_add(additional as u64)
+        .ok_or_else(|| Error::invalid_input("merged BloomFilter payload size overflowed u64"))?;
+    if total > MAX_BINARY_VALUE_BUFFER_LEN {
+        return Err(Error::invalid_input(format!(
+            "merged BloomFilter payload is {total} bytes, exceeding the Arrow BinaryArray limit \
+             of {MAX_BINARY_VALUE_BUFFER_LEN} bytes; merge fewer segments at a time"
+        )));
+    }
+    Ok(total)
+}
+
+/// Merge caller-selected BloomFilter segments into one self-contained segment.
+pub async fn merge_bloomfilter_indices(
+    source_indices: &[(&BloomFilterIndex, &RoaringBitmap)],
+    dest_store: &dyn IndexStore,
+) -> Result<CreatedIndex> {
+    let first = source_indices
+        .iter()
+        .find(|(_, fragment_filter)| !fragment_filter.is_empty())
+        .or_else(|| source_indices.first())
+        .ok_or_else(|| {
+            Error::invalid_input("merge_bloomfilter_indices requires at least one source index")
+        })?;
+    let params = BloomFilterIndexBuilderParams {
+        number_of_items: first.0.number_of_items,
+        probability: first.0.probability,
+    };
+
+    let mut blocks = Vec::new();
+    let mut merged_null_rows = RowAddrTreeMap::new();
+    let mut has_missing_null_bitmap = false;
+    let mut serialized_bytes = 0u64;
+    for (source, fragment_filter) in source_indices {
+        if fragment_filter.is_empty() {
+            continue;
+        }
+        if source.number_of_items != params.number_of_items
+            || source.probability != params.probability
+        {
+            return Err(Error::invalid_input(format!(
+                "cannot merge BloomFilter segments with different parameters: \
+                 number_of_items={}, probability={} and number_of_items={}, probability={}",
+                params.number_of_items,
+                params.probability,
+                source.number_of_items,
+                source.probability
+            )));
+        }
+        let source_zones = source.zones.iter().flat_map(|block| {
+            source.frag_reuse_index.as_deref().map_or_else(
+                || vec![block.clone()],
+                |remapper| remap_zone(block, remapper),
+            )
+        });
+        for block in source_zones.filter(|block| {
+            u32::try_from(block.bound.fragment_id)
+                .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id))
+        }) {
+            serialized_bytes =
+                checked_merged_payload_size(serialized_bytes, block.bloom_filter.to_bytes().len())?;
+            blocks.push(block);
+        }
+        match &source.null_rows {
+            Some(null_rows) => {
+                let mut filtered = source.frag_reuse_index.as_deref().map_or_else(
+                    || null_rows.clone(),
+                    |remapper| remapper.remap_row_addrs_tree_map(null_rows),
+                );
+                filtered.retain_fragments(fragment_filter.iter());
+                merged_null_rows |= &filtered;
+            }
+            None => has_missing_null_bitmap = true,
+        }
+    }
+    blocks.sort_by_key(|block| (block.bound.fragment_id, block.bound.start));
+
+    let mut builder = BloomFilterIndexBuilder::try_new(params)?;
+    builder.blocks = blocks;
+    if !has_missing_null_bitmap {
+        builder.null_rows = Some(merged_null_rows);
+    }
+    let files = builder.write_index(dest_store).await?;
+
+    Ok(CreatedIndex {
+        index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?,
+        index_version: BLOOMFILTER_INDEX_VERSION,
+        files,
+    })
+}
+
 fn default_number_of_items() -> u64 {
     *DEFAULT_NUMBER_OF_ITEMS
 }
@@ -542,6 +725,10 @@ impl BloomFilterIndexBuilderParams {
 pub struct BloomFilterIndexBuilder {
     params: BloomFilterIndexBuilderParams,
     blocks: Vec<BloomFilterStatistics>,
+    // None means "legacy index — null positions unknown"; Some means a complete bitmap.
+    // write_index omits the null-bitmap global buffer when this is None, preserving the
+    // legacy format so that downstream searches remain conservative.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl BloomFilterIndexBuilder {
@@ -549,6 +736,7 @@ impl BloomFilterIndexBuilder {
         Ok(Self {
             params,
             blocks: Vec::new(),
+            null_rows: None,
         })
     }
 
@@ -558,7 +746,9 @@ impl BloomFilterIndexBuilder {
     pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> {
         let processor = BloomFilterProcessor::new(self.params.clone())?;
         let trainer = ZoneTrainer::new(processor, self.params.number_of_items)?;
-        self.blocks = trainer.train(batches_source).await?;
+        let (blocks, null_rows) = trainer.train(batches_source).await?;
+        self.blocks = blocks;
+        self.null_rows = Some(null_rows);
         Ok(())
     }
 
@@ -615,7 +805,7 @@ impl BloomFilterIndexBuilder {
         Ok(RecordBatch::try_new(schema, columns)?)
     }
 
-    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<IndexFile> {
+    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
         let record_batch = self.bloomfilter_stats_as_batch()?;
 
         let mut file_schema = record_batch.schema().as_ref().clone();
@@ -623,7 +813,6 @@ impl BloomFilterIndexBuilder {
             BLOOMFILTER_ITEM_META_KEY.to_string(),
             self.params.number_of_items.to_string(),
         );
-
         file_schema.metadata.insert(
             BLOOMFILTER_PROBABILITY_META_KEY.to_string(),
             self.params.probability.to_string(),
@@ -633,7 +822,24 @@ impl BloomFilterIndexBuilder {
             .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema))
             .await?;
         index_file.write_record_batch(record_batch).await?;
-        index_file.finish().await
+
+        let bloomfilter_file = if let Some(null_rows) = self.null_rows {
+            let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size());
+            null_rows.serialize_into(&mut null_bitmap_bytes)?;
+            let null_bitmap_idx = index_file
+                .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes))
+                .await?;
+            index_file
+                .finish_with_metadata(HashMap::from([(
+                    NULL_BITMAP_META_KEY.to_string(),
+                    null_bitmap_idx.to_string(),
+                )]))
+                .await?
+        } else {
+            index_file.finish_with_metadata(HashMap::new()).await?
+        };
+
+        Ok(vec![bloomfilter_file])
     }
 }
 
@@ -980,7 +1186,7 @@ impl BloomFilterIndexPlugin {
         batches_source: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         options: Option<BloomFilterIndexBuilderParams>,
-    ) -> Result<IndexFile> {
+    ) -> Result<Vec<IndexFile>> {
         let mut builder = BloomFilterIndexBuilder::try_new(options.unwrap_or_default())?;
 
         builder.train(batches_source).await?;
@@ -1049,15 +1255,9 @@ impl BasicTrainer for BloomFilterIndexPlugin {
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         request: Box<dyn TrainingRequest>,
-        fragment_ids: Option<Vec<u32>>,
+        _fragment_ids: Option<Vec<u32>>,
         _progress: Arc<dyn crate::progress::IndexBuildProgress>,
     ) -> Result<CreatedIndex> {
-        if fragment_ids.is_some() {
-            return Err(Error::invalid_input_source(
-                "BloomFilter index does not support fragment training".into(),
-            ));
-        }
-
         let request = (request as Box<dyn std::any::Any>)
             .downcast::<BloomFilterIndexTrainingRequest>()
             .map_err(|_| {
@@ -1065,12 +1265,12 @@ impl BasicTrainer for BloomFilterIndexPlugin {
                     "must provide training request created by new_training_request".into(),
                 )
             })?;
-        let file = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?;
+        let files = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
                 .unwrap(),
             index_version: BLOOMFILTER_INDEX_VERSION,
-            files: vec![file],
+            files,
         })
     }
 }
@@ -1154,7 +1354,9 @@ impl TrainingRequest for BloomFilterIndexTrainingRequest {
 
 #[cfg(test)]
 mod tests {
+    use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle};
     use crate::scalar::registry::VALUE_COLUMN_NAME;
+    use std::collections::HashMap;
     use std::sync::Arc;
 
     use crate::scalar::bloomfilter::BloomFilterIndexPlugin;
@@ -1169,8 +1371,11 @@ mod tests {
     use lance_select::RowAddrTreeMap;
 
     use crate::scalar::{
-        BloomFilterQuery, ScalarIndex, SearchResult,
-        bloomfilter::{BloomFilterIndex, BloomFilterIndexBuilderParams},
+        BloomFilterQuery, IndexStore, ScalarIndex, SearchResult,
+        bloomfilter::{
+            BloomFilterIndex, BloomFilterIndexBuilderParams, MAX_BINARY_VALUE_BUFFER_LEN,
+            checked_merged_payload_size, merge_bloomfilter_indices,
+        },
         lance_format::LanceIndexStore,
     };
 
@@ -1178,6 +1383,16 @@ mod tests {
     use crate::metrics::NoOpMetricsCollector;
     use roaring::RoaringBitmap; // Import RoaringBitmap for the test
 
+    #[test]
+    fn test_checked_merged_payload_size_rejects_binary_overflow() {
+        assert_eq!(
+            checked_merged_payload_size(MAX_BINARY_VALUE_BUFFER_LEN - 1, 1).unwrap(),
+            MAX_BINARY_VALUE_BUFFER_LEN
+        );
+        let err = checked_merged_payload_size(MAX_BINARY_VALUE_BUFFER_LEN, 1).unwrap_err();
+        assert!(err.to_string().contains("Arrow BinaryArray limit"));
+    }
+
     // Adds a _rowaddr column emulating each batch as a new fragment
     fn add_row_addr(stream: SendableRecordBatchStream) -> SendableRecordBatchStream {
         let schema = stream.schema();
@@ -2037,10 +2252,10 @@ mod tests {
         expected.insert_range(500..750); // Should match the zone containing 500
         assert_eq!(result, SearchResult::at_most(expected));
 
-        // Test IsNull query
+        // Test IsNull query (no nulls in data, should return exact empty set)
         let query = BloomFilterQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); // No nulls in the data
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
 
         // Test IsIn query
         let query = BloomFilterQuery::IsIn(vec![
@@ -2133,4 +2348,342 @@ mod tests {
             _ => panic!("Expected AtMost search result from bloomfilter"),
         }
     }
+
+    // Writes a bloomfilter file in the legacy format (no null bitmap global buffer),
+    // simulating an index created before the null bitmap feature was added.
+    async fn write_legacy_bloomfilter(store: &dyn IndexStore, has_null: bool) {
+        use crate::scalar::bloomfilter::{
+            BLOOMFILTER_FILENAME, BLOOMFILTER_ITEM_META_KEY, BLOOMFILTER_PROBABILITY_META_KEY,
+        };
+        use arrow_array::BooleanArray;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+            Field::new("has_null", DataType::Boolean, false),
+            Field::new("bloom_filter_data", DataType::Binary, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![3u64])) as _,
+                Arc::new(BooleanArray::from(vec![has_null])) as _,
+                Arc::new(arrow_array::BinaryArray::from_vec(vec![b"".as_ref()])) as _,
+            ],
+        )
+        .unwrap();
+        let mut file_schema = schema.as_ref().clone();
+        file_schema
+            .metadata
+            .insert(BLOOMFILTER_ITEM_META_KEY.to_string(), "1000".to_string());
+        file_schema.metadata.insert(
+            BLOOMFILTER_PROBABILITY_META_KEY.to_string(),
+            "0.01".to_string(),
+        );
+        let mut writer = store
+            .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema))
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    // Updating a legacy (null_rows = None) index must not silently treat None as
+    // "no nulls".  The bug: `self.null_rows.clone().unwrap_or_default()` collapses
+    // None into an empty RowAddrTreeMap; after the merge the updated index has
+    // `null_rows = Some(empty)`, so an IsNull search returns `exact(empty)` — a
+    // false negative even though the legacy zone recorded has_null = true.
+    #[tokio::test]
+    async fn test_update_legacy_none_null_rows_not_treated_as_no_nulls() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Write a legacy-format index (no null bitmap) with has_null=true in its zone.
+        write_legacy_bloomfilter(store.as_ref(), true).await;
+
+        let index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert!(
+            index.null_rows.is_none(),
+            "precondition: legacy null_rows is None"
+        );
+
+        // Update with new data from fragment 1 (no nulls).  The destination is the
+        // same store so we can reload from it afterwards.
+        let new_schema = Arc::new(Schema::new(vec![
+            Field::new(VALUE_COLUMN_NAME, DataType::Int32, true),
+            Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let new_batch = RecordBatch::try_new(
+            new_schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int32Array::from(vec![
+                    Some(10i32),
+                    Some(20),
+                    Some(30),
+                ])) as _,
+                Arc::new(UInt64Array::from_iter_values(
+                    (0u64..3).map(|i| (1u64 << 32) | i),
+                )) as _,
+            ],
+        )
+        .unwrap();
+        let new_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            new_schema,
+            stream::once(std::future::ready(Ok(new_batch))),
+        ));
+
+        index
+            .update(new_stream, store.as_ref(), None)
+            .await
+            .unwrap();
+
+        let updated_index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // The legacy zone had has_null=true, so there ARE nulls at unknown positions.
+        // An IsNull search on the updated index must NOT claim "no nulls" (exact empty).
+        // It must be conservative and return AtMost, falling back to the has_null scan.
+        let result = updated_index
+            .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+
+        // With the bug: null_rows = Some(empty) → returns exact(empty) ← FALSE NEGATIVE
+        // With the fix: null_rows = None        → falls through to has_null scan → AtMost
+        assert!(
+            !result.is_exact(),
+            "IsNull on an updated legacy index must not return exact(empty); \
+             the legacy zone had has_null=true so nulls exist at unknown positions"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_legacy_bloomfilter_no_null_bitmap() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_legacy_bloomfilter(store.as_ref(), true).await;
+
+        let index = BloomFilterIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .expect("failed to load legacy bloomfilter");
+
+        assert!(
+            index.null_rows.is_none(),
+            "legacy index should have no null bitmap"
+        );
+
+        // IS NULL should fall back to the has_null zone scan and return AtMost, not Exact.
+        let result = index
+            .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(
+            !result.is_exact(),
+            "IS NULL on a legacy index should not be exact"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_bloomfilter_indices_preserves_exact_and_legacy_nulls() {
+        fn create_store() -> (TempObjDir, Arc<LanceIndexStore>) {
+            let tmpdir = TempObjDir::default();
+            let store = Arc::new(LanceIndexStore::new(
+                Arc::new(ObjectStore::local()),
+                tmpdir.clone(),
+                Arc::new(LanceCache::no_cache()),
+            ));
+            (tmpdir, store)
+        }
+
+        let (_first_tmpdir, first_store) = create_store();
+        let (_second_tmpdir, second_store) = create_store();
+        let (_merged_tmpdir, merged_store) = create_store();
+        let params = BloomFilterIndexBuilderParams::new(1000, 0.01);
+        for (fragment_id, store) in [(0_u64, &first_store), (1_u64, &second_store)] {
+            let row_base = fragment_id << 32;
+            let batch = record_batch!(
+                (VALUE_COLUMN_NAME, Int64, [Some(10), None, Some(30)]),
+                (ROW_ADDR, UInt64, [row_base, row_base + 1, row_base + 2])
+            )
+            .unwrap();
+            let schema = batch.schema();
+            let stream = Box::pin(RecordBatchStreamAdapter::new(
+                schema,
+                stream::once(async move { Ok(batch) }),
+            ));
+            BloomFilterIndexPlugin::train_bloomfilter_index(
+                stream,
+                store.as_ref(),
+                Some(params.clone()),
+            )
+            .await
+            .unwrap();
+        }
+
+        let first = BloomFilterIndex::load(first_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let second = BloomFilterIndex::load(second_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        merge_bloomfilter_indices(
+            &[
+                (first.as_ref(), &RoaringBitmap::from_iter([0])),
+                (second.as_ref(), &RoaringBitmap::from_iter([1])),
+            ],
+            merged_store.as_ref(),
+        )
+        .await
+        .unwrap();
+        let merged = BloomFilterIndex::load(merged_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let mut expected_nulls = RowAddrTreeMap::new();
+        expected_nulls.insert(1);
+        expected_nulls.insert((1_u64 << 32) + 1);
+        assert_eq!(
+            merged
+                .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            SearchResult::exact(expected_nulls)
+        );
+
+        let remapped_base = 2_u64 << 32;
+        let remapper = FragReuseIndexHandle(Arc::new(FragReuseIndex::new(
+            uuid::Uuid::new_v4(),
+            vec![HashMap::from([
+                (0, Some(remapped_base)),
+                (1, Some(remapped_base + 1)),
+                (2, Some(remapped_base + 2)),
+            ])],
+            FragReuseIndexDetails { versions: vec![] },
+        )));
+        let remapped_first = BloomFilterIndex::load(
+            first_store,
+            Some(Arc::new(remapper)),
+            &LanceCache::no_cache(),
+        )
+        .await
+        .unwrap();
+        let (_remapped_tmpdir, remapped_store) = create_store();
+        merge_bloomfilter_indices(
+            &[(remapped_first.as_ref(), &RoaringBitmap::from_iter([2]))],
+            remapped_store.as_ref(),
+        )
+        .await
+        .unwrap();
+        let remapped = BloomFilterIndex::load(remapped_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let mut expected_remapped_nulls = RowAddrTreeMap::new();
+        expected_remapped_nulls.insert(remapped_base + 1);
+        assert_eq!(
+            remapped
+                .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            SearchResult::exact(expected_remapped_nulls)
+        );
+        let candidates = remapped
+            .search(
+                &BloomFilterQuery::Equals(ScalarValue::Int64(Some(10))),
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        assert!(
+            candidates
+                .row_addrs()
+                .true_rows()
+                .row_addrs()
+                .unwrap()
+                .map(u64::from)
+                .any(|row_id| row_id == remapped_base)
+        );
+
+        let (_legacy_tmpdir, legacy_store) = create_store();
+        let (_legacy_merged_tmpdir, legacy_merged_store) = create_store();
+        write_legacy_bloomfilter(legacy_store.as_ref(), true).await;
+        let legacy = BloomFilterIndex::load(legacy_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        merge_bloomfilter_indices(
+            &[
+                (legacy.as_ref(), &RoaringBitmap::from_iter([0])),
+                (second.as_ref(), &RoaringBitmap::from_iter([1])),
+            ],
+            legacy_merged_store.as_ref(),
+        )
+        .await
+        .unwrap();
+        let legacy_merged =
+            BloomFilterIndex::load(legacy_merged_store, None, &LanceCache::no_cache())
+                .await
+                .unwrap();
+        assert!(
+            !legacy_merged
+                .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+                .await
+                .unwrap()
+                .is_exact()
+        );
+
+        let (_trimmed_tmpdir, trimmed_store) = create_store();
+        let mut ignored_legacy = legacy.as_ref().clone();
+        ignored_legacy.probability = 0.5;
+        merge_bloomfilter_indices(
+            &[
+                (&ignored_legacy, &RoaringBitmap::new()),
+                (second.as_ref(), &RoaringBitmap::from_iter([1])),
+            ],
+            trimmed_store.as_ref(),
+        )
+        .await
+        .unwrap();
+        let trimmed = BloomFilterIndex::load(trimmed_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let mut expected_nulls = RowAddrTreeMap::new();
+        expected_nulls.insert((1_u64 << 32) + 1);
+        assert_eq!(
+            trimmed
+                .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            SearchResult::exact(expected_nulls)
+        );
+
+        let (_empty_tmpdir, empty_store) = create_store();
+        merge_bloomfilter_indices(
+            &[(&ignored_legacy, &RoaringBitmap::new())],
+            empty_store.as_ref(),
+        )
+        .await
+        .unwrap();
+        let empty = BloomFilterIndex::load(empty_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(
+            empty
+                .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            SearchResult::exact(RowAddrTreeMap::new())
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/btree.rs b/rust/lance-index/src/scalar/btree.rs
index 7201574d58a..5bf89dfa29f 100644
--- a/rust/lance-index/src/scalar/btree.rs
+++ b/rust/lance-index/src/scalar/btree.rs
@@ -397,6 +397,8 @@ impl Ord for OrderableScalarValue {
                 panic!("Attempt to compare List with non-List")
             }
             (LargeList(_), _) => todo!(),
+            (ListView(_), _) => todo!(),
+            (LargeListView(_), _) => todo!(),
             (Map(_), Map(_)) => todo!(),
             (Map(left), Null) => {
                 if left.is_null(0) {
@@ -2562,9 +2564,7 @@ pub async fn train_btree_index(
     Ok(vec![pages_file, lookup_file])
 }
 
-fn find_single_partition_files(
-    files: &[lance_table::format::IndexFile],
-) -> Result<Option<(&str, &str)>> {
+fn find_single_partition_files(files: &[super::IndexFile]) -> Result<Option<(&str, &str)>> {
     let lookup_files = files
         .iter()
         .filter_map(|file| {
@@ -5327,12 +5327,8 @@ mod tests {
             mapping.insert(old_id, None);
         }
 
-        let mut new_id_counter = 100_000;
-
         // Remap all other rows
-        for old_id in (0..1000).chain(10000..15000) {
-            let new_id = new_id_counter;
-            new_id_counter += 1;
+        for (new_id, old_id) in (100_000..).zip((0..1000).chain(10000..15000)) {
             mapping.insert(old_id, Some(new_id));
         }
 
@@ -6289,7 +6285,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_btree_index_state_reconstruct_applies_frag_reuse_index() {
-        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails};
+        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle};
         use std::collections::HashMap;
         use uuid::Uuid;
 
@@ -6322,11 +6318,11 @@ mod tests {
         // Querying for value == 0 should now return row 5000, confirming reconstruct threaded
         // the FragReuseIndex through to the rebuilt BTreeIndex.
         let frag_reuse_index: Arc<dyn crate::scalar::RowIdRemapper> =
-            Arc::new(FragReuseIndex::new(
+            Arc::new(FragReuseIndexHandle(Arc::new(FragReuseIndex::new(
                 Uuid::new_v4(),
                 vec![HashMap::from([(0u64, Some(5000u64))])],
                 FragReuseIndexDetails { versions: vec![] },
-            ));
+            ))));
         let reconstructed = state
             .reconstruct(
                 test_store.clone(),
diff --git a/rust/lance-index/src/scalar/btree/flat.rs b/rust/lance-index/src/scalar/btree/flat.rs
index 7eb9f0422f3..1af5a6a69e9 100644
--- a/rust/lance-index/src/scalar/btree/flat.rs
+++ b/rust/lance-index/src/scalar/btree/flat.rs
@@ -218,13 +218,13 @@ impl FlatIndex {
                         ));
                     }
                 }
-                (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) => {
-                    if lower.is_null() {
-                        return Ok(NullableRowAddrSet::new(
-                            Default::default(),
-                            self.all_addrs_map.clone(),
-                        ));
-                    }
+                (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded)
+                    if lower.is_null() =>
+                {
+                    return Ok(NullableRowAddrSet::new(
+                        Default::default(),
+                        self.all_addrs_map.clone(),
+                    ));
                 }
                 _ => {}
             },
diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs
index e495d5b701f..6fe635948cb 100644
--- a/rust/lance-index/src/scalar/expression.rs
+++ b/rust/lance-index/src/scalar/expression.rs
@@ -1,7 +1,12 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{ops::Bound, sync::Arc};
+use std::{
+    cmp::Ordering,
+    collections::{HashMap, HashSet, hash_map::Entry},
+    ops::Bound,
+    sync::Arc,
+};
 
 use arrow_schema::{DataType, Field};
 use async_recursion::async_recursion;
@@ -1402,6 +1407,20 @@ pub trait ScalarIndexLoader: Send + Sync {
         index_name: &str,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn ScalarIndex>>;
+
+    /// Translate an address-domain index result into the row-id domain
+    ///
+    /// Address-domain indices (see [`ScalarIndex::results_are_row_addresses`])
+    /// report matches as physical row addresses. The default returns `result`
+    /// unchanged, which is correct when addresses and row ids coincide (no
+    /// stable row ids). A dataset with stable row ids overrides this to remap
+    /// addresses to stable row ids via its per-fragment row-id sequences.
+    async fn row_addr_result_to_row_ids(
+        &self,
+        result: NullableIndexExprResult,
+    ) -> Result<NullableIndexExprResult> {
+        Ok(result)
+    }
 }
 
 /// This represents a search into a scalar index
@@ -1462,173 +1481,389 @@ impl PartialEq for ScalarIndexExpr {
     }
 }
 
+/// Conservative grouping key for rewrites targeting the same parsed scalar index.
+/// `index_type` is display metadata, but including it prevents rewrites across
+/// index implementations that may not share query semantics.
+type ScalarIndexQueryKey = (String, String, String);
+
 /// Returns the tighter (more restrictive) lower bound.
-/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value.
-fn tighter_lower_bound(a: &Bound<ScalarValue>, b: &Bound<ScalarValue>) -> Bound<ScalarValue> {
+///
+/// Returns `None` if the bound values cannot be compared. In that case callers
+/// keep the original predicates rather than inventing ordering semantics.
+fn tighter_lower_bound(
+    a: &Bound<ScalarValue>,
+    b: &Bound<ScalarValue>,
+) -> Option<Bound<ScalarValue>> {
     match (a, b) {
-        (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(),
-        (Bound::Included(va), Bound::Included(vb)) => {
-            if va >= vb {
-                Bound::Included(va.clone())
-            } else {
-                Bound::Included(vb.clone())
-            }
-        }
-        (Bound::Excluded(va), Bound::Excluded(vb)) => {
-            if va >= vb {
-                Bound::Excluded(va.clone())
-            } else {
-                Bound::Excluded(vb.clone())
-            }
-        }
-        (Bound::Excluded(va), Bound::Included(vb)) => {
-            if va >= vb {
-                Bound::Excluded(va.clone())
-            } else {
-                Bound::Included(vb.clone())
-            }
-        }
-        (Bound::Included(va), Bound::Excluded(vb)) => {
-            if vb >= va {
-                Bound::Excluded(vb.clone())
-            } else {
-                Bound::Included(va.clone())
-            }
-        }
+        (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded),
+        (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()),
+        (
+            Bound::Included(a_value) | Bound::Excluded(a_value),
+            Bound::Included(b_value) | Bound::Excluded(b_value),
+        ) => match a_value.partial_cmp(b_value)? {
+            Ordering::Less => Some(b.clone()),
+            Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)),
+            Ordering::Greater => Some(a.clone()),
+        },
     }
 }
 
 /// Returns the tighter (more restrictive) upper bound.
-/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value.
-fn tighter_upper_bound(a: &Bound<ScalarValue>, b: &Bound<ScalarValue>) -> Bound<ScalarValue> {
+///
+/// Returns `None` if the bound values cannot be compared. In that case callers
+/// keep the original predicates rather than inventing ordering semantics.
+fn tighter_upper_bound(
+    a: &Bound<ScalarValue>,
+    b: &Bound<ScalarValue>,
+) -> Option<Bound<ScalarValue>> {
     match (a, b) {
-        (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(),
-        (Bound::Included(va), Bound::Included(vb)) => {
-            if va <= vb {
-                Bound::Included(va.clone())
-            } else {
-                Bound::Included(vb.clone())
-            }
-        }
-        (Bound::Excluded(va), Bound::Excluded(vb)) => {
-            if va <= vb {
-                Bound::Excluded(va.clone())
-            } else {
-                Bound::Excluded(vb.clone())
+        (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded),
+        (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()),
+        (
+            Bound::Included(a_value) | Bound::Excluded(a_value),
+            Bound::Included(b_value) | Bound::Excluded(b_value),
+        ) => match a_value.partial_cmp(b_value)? {
+            Ordering::Less => Some(a.clone()),
+            Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)),
+            Ordering::Greater => Some(b.clone()),
+        },
+    }
+}
+
+fn is_excluded_bound(bound: &Bound<ScalarValue>) -> bool {
+    matches!(bound, Bound::Excluded(_))
+}
+
+/// For an equal scalar value, an excluded bound is stricter than an included
+/// bound. This handles cases like `x >= 5 AND x > 5`.
+fn stricter_bound_for_equal_value(
+    value: &ScalarValue,
+    lhs: &Bound<ScalarValue>,
+    rhs: &Bound<ScalarValue>,
+) -> Bound<ScalarValue> {
+    if is_excluded_bound(lhs) || is_excluded_bound(rhs) {
+        Bound::Excluded(value.clone())
+    } else {
+        Bound::Included(value.clone())
+    }
+}
+
+fn range_has_non_null_bound(lower: &Bound<ScalarValue>, upper: &Bound<ScalarValue>) -> bool {
+    let mut has_bound = false;
+    for bound in [lower, upper] {
+        match bound {
+            Bound::Included(value) | Bound::Excluded(value) => {
+                if value.is_null() {
+                    return false;
+                }
+                has_bound = true;
             }
+            Bound::Unbounded => {}
         }
-        (Bound::Excluded(va), Bound::Included(vb)) => {
-            if va <= vb {
-                Bound::Excluded(va.clone())
-            } else {
-                Bound::Included(vb.clone())
-            }
+    }
+    has_bound
+}
+
+/// Null bounds are skipped by range optimization. Comparisons with NULL should
+/// already be rejected by normal parsing, but this keeps manually constructed
+/// ScalarIndexExpr values from being rewritten into misleading ranges.
+fn range_has_null_bound(lower: &Bound<ScalarValue>, upper: &Bound<ScalarValue>) -> bool {
+    [lower, upper].iter().any(|bound| match bound {
+        Bound::Included(value) | Bound::Excluded(value) => value.is_null(),
+        Bound::Unbounded => false,
+    })
+}
+
+impl ScalarIndexSearch {
+    /// Only SargableQuery participates in these expression-level rewrites.
+    /// Other AnyQuery implementations may have different null/range semantics.
+    fn sargable_query(&self) -> Option<&SargableQuery> {
+        self.query.as_any().downcast_ref::<SargableQuery>()
+    }
+
+    /// Require a concrete SargableQuery before producing a key so callers do not
+    /// accidentally group unrelated AnyQuery implementations by metadata alone.
+    fn sargable_query_key(&self) -> Option<ScalarIndexQueryKey> {
+        self.sargable_query()?;
+        Some((
+            self.column.clone(),
+            self.index_name.clone(),
+            self.index_type.clone(),
+        ))
+    }
+
+    /// Only exact SargableQuery values may drive NULL-elimination rewrites.
+    /// Range merging also accepts inexact queries and preserves their recheck.
+    fn exact_sargable_query(&self) -> Option<&SargableQuery> {
+        if self.needs_recheck {
+            return None;
         }
-        (Bound::Included(va), Bound::Excluded(vb)) => {
-            if vb <= va {
-                Bound::Excluded(vb.clone())
-            } else {
-                Bound::Included(va.clone())
+        self.sargable_query()
+    }
+
+    /// Return a scalar-index key only when the query is an exact SargableQuery.
+    fn exact_sargable_query_key(&self) -> Option<ScalarIndexQueryKey> {
+        self.exact_sargable_query()?;
+        self.sargable_query_key()
+    }
+
+    /// A query is null-intolerant if the original predicate cannot match NULL.
+    /// Only exact queries can make `IS NOT NULL` redundant in the index expression.
+    fn is_null_intolerant_sargable_query(&self) -> bool {
+        match self.exact_sargable_query() {
+            Some(SargableQuery::Range(lower, upper)) => range_has_non_null_bound(lower, upper),
+            Some(SargableQuery::Equals(value)) => !value.is_null(),
+            Some(SargableQuery::IsIn(values)) => {
+                !values.is_empty() && values.iter().all(|value| !value.is_null())
             }
+            _ => false,
         }
     }
 }
 
 impl ScalarIndexExpr {
-    /// Optimize the expression tree by merging range queries on the same index.
+    /// Apply scalar-index optimizer rules within each contiguous AND region.
+    ///
+    /// Range queries for the same parsed scalar index are intersected and retain
+    /// `needs_recheck` if any input range requires it. An exact `IS NOT NULL`
+    /// query is removed when another exact same-index query is null-intolerant.
+    /// OR branches are optimized independently. Range queries are still merged
+    /// under NOT, but `IS NOT NULL` elimination is disabled there to preserve
+    /// SQL NULL semantics.
     ///
-    /// This collects all leaf Range queries from the AND tree, groups them by
-    /// index name, merges overlapping ranges into a single closed-range query,
-    /// and rebuilds the tree. This handles the case where `log_time >= X` and
-    /// `log_time <= Y` end up in different branches of a nested AND tree.
+    /// Compatible [`SargableQuery::Range`] values carried by
+    /// [`ScalarIndexSearch`] are merged into one query.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use std::{ops::Bound, sync::Arc};
+    /// # use lance_index::scalar::{SargableQuery, expression::{ScalarIndexExpr, ScalarIndexSearch}};
+    /// # let range = |lower, upper| ScalarIndexExpr::Query(ScalarIndexSearch {
+    /// #     column: "x".into(),
+    /// #     index_name: "x_idx".into(),
+    /// #     index_type: "BTree".into(),
+    /// #     query: Arc::new(SargableQuery::Range(lower, upper)),
+    /// #     needs_recheck: false,
+    /// #     fragment_bitmap: None,
+    /// # });
+    /// let optimized = ScalarIndexExpr::And(
+    ///     Box::new(range(Bound::Included(10_i64.into()), Bound::Unbounded)),
+    ///     Box::new(range(Bound::Unbounded, Bound::Included(20_i64.into()))),
+    /// )
+    /// .optimize();
+    ///
+    /// assert!(matches!(optimized, ScalarIndexExpr::Query(_)));
+    /// ```
     pub fn optimize(self) -> Self {
+        self.optimize_with_context(true)
+    }
+
+    fn optimize_with_context(self, is_not_null_elidable: bool) -> Self {
         match self {
-            Self::And(_, _) => self.optimize_and_tree(),
-            Self::Or(lhs, rhs) => Self::Or(Box::new(lhs.optimize()), Box::new(rhs.optimize())),
-            Self::Not(inner) => Self::Not(Box::new(inner.optimize())),
+            Self::And(_, _) => self.optimize_and_tree(is_not_null_elidable),
+            Self::Or(lhs, rhs) => Self::Or(
+                Box::new(lhs.optimize_with_context(is_not_null_elidable)),
+                Box::new(rhs.optimize_with_context(is_not_null_elidable)),
+            ),
+            Self::Not(inner) => Self::Not(Box::new(inner.optimize_with_context(false))),
             other => other,
         }
     }
 
-    /// Flatten an AND tree, merge ranges on same index, rebuild.
-    fn optimize_and_tree(self) -> Self {
+    /// Flatten one contiguous AND region, apply local optimizer rules, and
+    /// rebuild a balanced AND tree.
+    fn optimize_and_tree(self, is_not_null_elidable: bool) -> Self {
         let mut leaves = Vec::new();
-        self.collect_and_leaves(&mut leaves);
+        self.collect_and_leaves(&mut leaves, is_not_null_elidable);
+        let leaves = Self::optimize_and_leaves(leaves, is_not_null_elidable);
 
-        // Try to merge Range queries on the same index
-        let mut merged_indices: Vec<bool> = vec![false; leaves.len()];
-        let mut result_leaves: Vec<Self> = Vec::new();
+        Self::rebuild_and_tree(leaves).expect("AND tree optimization should keep at least one leaf")
+    }
+
+    /// Apply optimizer rules within one AND region only. OR branches have already
+    /// been kept as opaque leaves, so range and null-intolerance rewrites cannot
+    /// cross boolean boundaries.
+    fn optimize_and_leaves(leaves: Vec<Self>, is_not_null_elidable: bool) -> Vec<Self> {
+        let null_intolerant_keys = if !is_not_null_elidable {
+            HashSet::new()
+        } else {
+            let is_not_null_keys = leaves
+                .iter()
+                .filter_map(Self::is_not_null_query_key)
+                .collect::<HashSet<_>>();
+            if is_not_null_keys.is_empty() {
+                HashSet::new()
+            } else {
+                leaves
+                    .iter()
+                    .filter_map(Self::null_intolerant_query_key)
+                    .filter(|key| is_not_null_keys.contains(key))
+                    .collect()
+            }
+        };
 
-        for i in 0..leaves.len() {
-            if merged_indices[i] {
+        let mut optimized = Vec::with_capacity(leaves.len());
+        let mut range_positions = HashMap::new();
+
+        for leaf in leaves {
+            if let Some(key) = leaf.is_not_null_query_key()
+                && null_intolerant_keys.contains(&key)
+            {
                 continue;
             }
-            let mut current = leaves[i].clone();
 
-            // Try to merge with subsequent leaves on the same index
-            for j in (i + 1)..leaves.len() {
-                if merged_indices[j] {
-                    continue;
-                }
-                if let Some(merged) = try_merge_range_pair(&current, &leaves[j]) {
-                    current = merged;
-                    merged_indices[j] = true;
+            if let Some(key) = leaf.range_query_key() {
+                match range_positions.entry(key) {
+                    Entry::Vacant(entry) => {
+                        entry.insert(optimized.len());
+                        optimized.push(leaf);
+                    }
+                    Entry::Occupied(entry) => {
+                        if !optimized[*entry.get()].try_merge_range(&leaf) {
+                            optimized.push(leaf);
+                        }
+                    }
                 }
+            } else {
+                optimized.push(leaf);
             }
-
-            result_leaves.push(current);
         }
 
-        // Rebuild the AND tree from remaining leaves
-        let mut iter = result_leaves.into_iter();
-        let first = iter.next().expect("AND tree must have at least one leaf");
-        iter.fold(first, |acc, leaf| Self::And(Box::new(acc), Box::new(leaf)))
+        optimized
     }
 
-    /// Recursively collect all leaf nodes from an AND tree.
-    fn collect_and_leaves(self, leaves: &mut Vec<Self>) {
+    /// Recursively collect all leaf nodes from an AND tree. OR branches are
+    /// optimized independently with the current null-elision context. NOT
+    /// branches remain leaves while their children are optimized with null
+    /// elimination disabled.
+    fn collect_and_leaves(self, leaves: &mut Vec<Self>, is_not_null_elidable: bool) {
         match self {
             Self::And(lhs, rhs) => {
-                lhs.collect_and_leaves(leaves);
-                rhs.collect_and_leaves(leaves);
+                lhs.collect_and_leaves(leaves, is_not_null_elidable);
+                rhs.collect_and_leaves(leaves, is_not_null_elidable);
+            }
+            Self::Or(lhs, rhs) => {
+                leaves.push(Self::Or(
+                    Box::new(lhs.optimize_with_context(is_not_null_elidable)),
+                    Box::new(rhs.optimize_with_context(is_not_null_elidable)),
+                ));
+            }
+            Self::Not(inner) => {
+                leaves.push(Self::Not(Box::new(inner.optimize_with_context(false))))
             }
             other => leaves.push(other),
         }
     }
-}
 
-/// Try to merge two ScalarIndexExpr nodes if they are both Range queries on the same index.
-fn try_merge_range_pair(lhs: &ScalarIndexExpr, rhs: &ScalarIndexExpr) -> Option<ScalarIndexExpr> {
-    let (ScalarIndexExpr::Query(l), ScalarIndexExpr::Query(r)) = (lhs, rhs) else {
-        return None;
-    };
-    if l.index_name != r.index_name || l.column != r.column {
-        return None;
+    /// Rebuild as a balanced tree so large planner-generated conjunctions do not
+    /// become deep left-leaning trees after optimization.
+    fn rebuild_and_tree(leaves: Vec<Self>) -> Option<Self> {
+        let mut leaves = leaves;
+        if leaves.is_empty() {
+            return None;
+        }
+
+        while leaves.len() > 1 {
+            let mut next = Vec::with_capacity(leaves.len().div_ceil(2));
+            let mut iter = leaves.into_iter();
+            while let Some(lhs) = iter.next() {
+                if let Some(rhs) = iter.next() {
+                    next.push(Self::And(Box::new(lhs), Box::new(rhs)));
+                } else {
+                    next.push(lhs);
+                }
+            }
+            leaves = next;
+        }
+
+        leaves.pop()
     }
 
-    let l_query = l.query.as_any().downcast_ref::<SargableQuery>()?;
-    let r_query = r.query.as_any().downcast_ref::<SargableQuery>()?;
+    /// Detect the scalar-index representation of `col IS NOT NULL`, which is
+    /// stored as `NOT(col IS NULL)`.
+    fn is_not_null_query_key(&self) -> Option<ScalarIndexQueryKey> {
+        match self {
+            Self::Not(inner) => match inner.as_ref() {
+                Self::Query(search)
+                    if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) =>
+                {
+                    search.exact_sargable_query_key()
+                }
+                _ => None,
+            },
+            _ => None,
+        }
+    }
 
-    let (SargableQuery::Range(l_low, l_high), SargableQuery::Range(r_low, r_high)) =
-        (l_query, r_query)
-    else {
-        return None;
-    };
+    /// Return the key of a same-column predicate that makes `IS NOT NULL`
+    /// redundant in the same AND region.
+    fn null_intolerant_query_key(&self) -> Option<ScalarIndexQueryKey> {
+        match self {
+            Self::Query(search) if search.is_null_intolerant_sargable_query() => {
+                search.exact_sargable_query_key()
+            }
+            Self::Not(inner) => match inner.as_ref() {
+                Self::Query(search)
+                    if matches!(
+                        search.exact_sargable_query(),
+                        Some(SargableQuery::Equals(value)) if !value.is_null()
+                    ) =>
+                {
+                    search.exact_sargable_query_key()
+                }
+                _ => None,
+            },
+            _ => None,
+        }
+    }
 
-    let merged_low = tighter_lower_bound(l_low, r_low);
-    let merged_high = tighter_upper_bound(l_high, r_high);
-
-    Some(ScalarIndexExpr::Query(ScalarIndexSearch {
-        column: l.column.clone(),
-        index_name: l.index_name.clone(),
-        index_type: l.index_type.clone(),
-        query: Arc::new(SargableQuery::Range(merged_low, merged_high)),
-        needs_recheck: l.needs_recheck || r.needs_recheck,
-        // Both queries target the same index (checked above), so they share
-        // the same fragment coverage; carry it over to keep the merged query
-        // usable by coverage-dependent optimizer rules.
-        fragment_bitmap: l.fragment_bitmap.clone(),
-    }))
+    /// Return the grouping key for an optimizable range query. Ranges with NULL
+    /// bounds are left unchanged.
+    fn range_query_key(&self) -> Option<ScalarIndexQueryKey> {
+        let Self::Query(search) = self else {
+            return None;
+        };
+        let SargableQuery::Range(lower, upper) = search.sargable_query()? else {
+            return None;
+        };
+        if range_has_null_bound(lower, upper) {
+            return None;
+        }
+        search.sargable_query_key()
+    }
+
+    /// Merge another compatible range into this expression.
+    ///
+    /// The caller must first group both expressions by [`ScalarIndexQueryKey`].
+    /// Different fragment coverage or incomparable bounds leave both predicates
+    /// unchanged. Empty intersections are retained as ranges.
+    fn try_merge_range(&mut self, other: &Self) -> bool {
+        let (Self::Query(search), Self::Query(other_search)) = (self, other) else {
+            return false;
+        };
+        if search.fragment_bitmap != other_search.fragment_bitmap {
+            return false;
+        }
+
+        let (
+            Some(SargableQuery::Range(lower, upper)),
+            Some(SargableQuery::Range(other_lower, other_upper)),
+        ) = (search.sargable_query(), other_search.sargable_query())
+        else {
+            return false;
+        };
+        let Some(lower) = tighter_lower_bound(lower, other_lower) else {
+            return false;
+        };
+        let Some(upper) = tighter_upper_bound(upper, other_upper) else {
+            return false;
+        };
+
+        search.query = Arc::new(SargableQuery::Range(lower, upper));
+        search.needs_recheck |= other_search.needs_recheck;
+        true
+    }
 }
 
 impl std::fmt::Display for ScalarIndexExpr {
@@ -1648,12 +1883,16 @@ impl std::fmt::Display for ScalarIndexExpr {
     }
 }
 
-impl From<SearchResult> for NullableIndexExprResult {
-    fn from(result: SearchResult) -> Self {
-        match result {
-            SearchResult::Exact(mask) => Self::exact(NullableRowAddrMask::AllowList(mask)),
-            SearchResult::AtMost(mask) => Self::at_most(NullableRowAddrMask::AllowList(mask)),
-            SearchResult::AtLeast(mask) => Self::at_least(NullableRowAddrMask::AllowList(mask)),
+fn search_result_to_nullable(result: SearchResult) -> NullableIndexExprResult {
+    match result {
+        SearchResult::Exact(mask) => {
+            NullableIndexExprResult::exact(NullableRowAddrMask::AllowList(mask))
+        }
+        SearchResult::AtMost(mask) => {
+            NullableIndexExprResult::at_most(NullableRowAddrMask::AllowList(mask))
+        }
+        SearchResult::AtLeast(mask) => {
+            NullableIndexExprResult::at_least(NullableRowAddrMask::AllowList(mask))
         }
     }
 }
@@ -1693,7 +1932,15 @@ impl ScalarIndexExpr {
                     .load_index(&search.column, &search.index_name, metrics)
                     .await?;
                 let search_result = index.search(search.query.as_ref(), metrics).await?;
-                Ok(search_result.into())
+                let result = search_result_to_nullable(search_result);
+                if index.results_are_row_addresses() {
+                    // Translate address-domain results to the row-id domain
+                    // before combining or scanning; otherwise stable-row-id
+                    // datasets silently drop matches (issue #7434).
+                    index_loader.row_addr_result_to_row_ids(result).await
+                } else {
+                    Ok(result)
+                }
             }
         }
     }
@@ -1838,7 +2085,7 @@ fn maybe_scalar(expr: &Expr, expected_type: &DataType) -> Option<ScalarValue> {
         // In this case we need to extract the value, apply the cast, and then test the casted value
         Expr::Cast(cast) => match cast.expr.as_ref() {
             Expr::Literal(value, _) => {
-                let casted = value.cast_to(&cast.data_type).ok()?;
+                let casted = value.cast_to(cast.field.data_type()).ok()?;
                 safe_coerce_scalar(&casted, expected_type)
             }
             _ => None,
@@ -2426,9 +2673,10 @@ mod tests {
         let state = ctx.state();
         let mut expr = state.create_logical_expr(expr, &df_schema).unwrap();
         if optimize {
-            let simplify_context = SimplifyContext::default()
+            let simplify_context = SimplifyContext::builder()
                 .with_schema(Arc::new(df_schema))
-                .with_query_execution_start_time(Some(Utc::now()));
+                .with_query_execution_start_time(Some(Utc::now()))
+                .build();
             let simplifier =
                 datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
             expr = simplifier.simplify(expr).unwrap();
@@ -3302,9 +3550,10 @@ mod tests {
             .unwrap();
 
         // Apply DataFusion simplification (this may convert starts_with to LIKE)
-        let simplify_context = SimplifyContext::default()
+        let simplify_context = SimplifyContext::builder()
             .with_schema(Arc::new(df_schema))
-            .with_query_execution_start_time(Some(Utc::now()));
+            .with_query_execution_start_time(Some(Utc::now()))
+            .build();
         let simplifier =
             datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
         let simplified_expr = simplifier.simplify(expr).unwrap();
@@ -3871,7 +4120,7 @@ mod tests {
         // and the other queries as separate leaves
         // Let's verify by collecting leaves
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
+        optimized.collect_and_leaves(&mut leaves, true);
 
         // Should have 3 leaves: fqdn, merged_time, channel
         assert_eq!(
@@ -3943,7 +4192,7 @@ mod tests {
 
         // Should remain as two separate leaves (not merged)
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
+        optimized.collect_and_leaves(&mut leaves, true);
         assert_eq!(leaves.len(), 2);
     }
 
@@ -3980,7 +4229,7 @@ mod tests {
         let optimized = expr.optimize();
 
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
+        optimized.collect_and_leaves(&mut leaves, true);
         assert_eq!(leaves.len(), 2, "Equals + Range should not merge");
     }
 
@@ -4020,7 +4269,7 @@ mod tests {
         let optimized = expr.optimize();
 
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
+        optimized.collect_and_leaves(&mut leaves, true);
         assert_eq!(leaves.len(), 1);
 
         if let ScalarIndexExpr::Query(s) = &leaves[0] {
@@ -4099,7 +4348,7 @@ mod tests {
         let optimized = expr.optimize();
 
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
+        optimized.collect_and_leaves(&mut leaves, true);
         // Should have 2 leaves: OR node (preserved) + merged range
         assert_eq!(leaves.len(), 2);
 
@@ -4129,52 +4378,638 @@ mod tests {
     }
 
     #[test]
-    fn test_optimize_needs_recheck_preserved() {
-        use super::{ScalarIndexExpr, ScalarIndexSearch};
-        use crate::scalar::SargableQuery;
-        use datafusion_common::ScalarValue;
-        use std::ops::Bound;
-        use std::sync::Arc;
+    fn test_optimize_respects_fragment_coverage_when_merging_ranges() {
+        let fragments = RoaringBitmap::from_iter([1, 3, 5]);
+        let lower = test_scalar_range_with_metadata(
+            Bound::Included(ScalarValue::Int64(Some(1))),
+            Bound::Unbounded,
+            true,
+            Some(fragments.clone()),
+        );
+        let upper = test_scalar_range_with_metadata(
+            Bound::Unbounded,
+            Bound::Included(ScalarValue::Int64(Some(99))),
+            false,
+            Some(fragments.clone()),
+        );
 
-        // If either range has needs_recheck=true, merged result should too
-        let range_a = ScalarIndexExpr::Query(ScalarIndexSearch {
-            column: "x".to_string(),
-            index_name: "idx_x".to_string(),
-            index_type: "".to_string(),
-            query: Arc::new(SargableQuery::Range(
-                Bound::Included(ScalarValue::Int64(Some(1))),
+        let leaves = collect_test_and_leaves(test_and_terms(vec![lower.clone(), upper]).optimize());
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Query(search)
+                if search.needs_recheck
+                    && search.fragment_bitmap.as_ref() == Some(&fragments)
+                    && matches!(
+                        search.sargable_query(),
+                        Some(SargableQuery::Range(
+                            Bound::Included(ScalarValue::Int64(Some(1))),
+                            Bound::Included(ScalarValue::Int64(Some(99))),
+                        ))
+                    )
+        ));
+
+        let upper_without_coverage = test_scalar_range_with_metadata(
+            Bound::Unbounded,
+            Bound::Included(ScalarValue::Int64(Some(99))),
+            false,
+            None,
+        );
+        for (lhs, rhs) in [
+            (lower.clone(), upper_without_coverage.clone()),
+            (upper_without_coverage, lower),
+        ] {
+            let optimized = ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs)).optimize();
+            let leaves = collect_test_and_leaves(optimized);
+            assert_eq!(leaves.len(), 2);
+            assert!(leaves.iter().any(|leaf| {
+                matches!(leaf, ScalarIndexExpr::Query(search)
+                    if search.needs_recheck
+                        && search.fragment_bitmap.as_ref() == Some(&fragments))
+            }));
+            assert!(leaves.iter().any(|leaf| {
+                matches!(leaf, ScalarIndexExpr::Query(search)
+                    if !search.needs_recheck && search.fragment_bitmap.is_none())
+            }));
+        }
+    }
+
+    #[test]
+    fn test_optimize_merges_recheck_ranges_into_empty_range() {
+        let lower = test_scalar_query_with_recheck(
+            "x",
+            "idx_x",
+            SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(200))),
                 Bound::Unbounded,
-            )),
+            ),
+        );
+        let upper = test_scalar_query_with_recheck(
+            "x",
+            "idx_x",
+            SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(100))),
+            ),
+        );
+
+        let leaves = collect_test_and_leaves(test_and_terms(vec![lower, upper]).optimize());
+
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Query(search)
+                if search.needs_recheck
+                    && matches!(
+                        search.sargable_query(),
+                        Some(SargableQuery::Range(
+                            Bound::Included(ScalarValue::Int64(Some(200))),
+                            Bound::Included(ScalarValue::Int64(Some(100))),
+                        ))
+                    )
+        ));
+    }
+
+    fn test_scalar_query(column: &str, index_name: &str, query: SargableQuery) -> ScalarIndexExpr {
+        ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: column.to_string(),
+            index_name: index_name.to_string(),
+            index_type: "BTree".to_string(),
+            query: Arc::new(query),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        })
+    }
+
+    fn test_scalar_query_with_recheck(
+        column: &str,
+        index_name: &str,
+        query: SargableQuery,
+    ) -> ScalarIndexExpr {
+        ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: column.to_string(),
+            index_name: index_name.to_string(),
+            index_type: "ZoneMap".to_string(),
+            query: Arc::new(query),
             needs_recheck: true,
             fragment_bitmap: None,
-        });
-        let range_b = ScalarIndexExpr::Query(ScalarIndexSearch {
+        })
+    }
+
+    fn test_scalar_range(
+        column: &str,
+        index_name: &str,
+        lower: Bound<ScalarValue>,
+        upper: Bound<ScalarValue>,
+    ) -> ScalarIndexExpr {
+        test_scalar_query(column, index_name, SargableQuery::Range(lower, upper))
+    }
+
+    fn test_scalar_range_with_metadata(
+        lower: Bound<ScalarValue>,
+        upper: Bound<ScalarValue>,
+        needs_recheck: bool,
+        fragment_bitmap: Option<RoaringBitmap>,
+    ) -> ScalarIndexExpr {
+        ScalarIndexExpr::Query(ScalarIndexSearch {
             column: "x".to_string(),
             index_name: "idx_x".to_string(),
-            index_type: "".to_string(),
-            query: Arc::new(SargableQuery::Range(
-                Bound::Unbounded,
-                Bound::Included(ScalarValue::Int64(Some(99))),
-            )),
-            needs_recheck: false,
-            fragment_bitmap: None,
-        });
+            index_type: "ZoneMap".to_string(),
+            query: Arc::new(SargableQuery::Range(lower, upper)),
+            needs_recheck,
+            fragment_bitmap,
+        })
+    }
 
-        let expr = ScalarIndexExpr::And(Box::new(range_a), Box::new(range_b));
-        let optimized = expr.optimize();
+    fn test_and_terms(mut terms: Vec<ScalarIndexExpr>) -> ScalarIndexExpr {
+        assert!(!terms.is_empty());
+        while terms.len() > 1 {
+            let mut next = Vec::with_capacity(terms.len().div_ceil(2));
+            let mut iter = terms.into_iter();
+            while let Some(lhs) = iter.next() {
+                if let Some(rhs) = iter.next() {
+                    next.push(ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs)));
+                } else {
+                    next.push(lhs);
+                }
+            }
+            terms = next;
+        }
+        terms.pop().unwrap()
+    }
 
+    fn collect_test_and_leaves(expr: ScalarIndexExpr) -> Vec<ScalarIndexExpr> {
         let mut leaves = Vec::new();
-        optimized.collect_and_leaves(&mut leaves);
-        assert_eq!(leaves.len(), 1);
+        expr.collect_and_leaves(&mut leaves, true);
+        leaves
+    }
 
-        if let ScalarIndexExpr::Query(s) = &leaves[0] {
-            assert!(
-                s.needs_recheck,
-                "Merged query should preserve needs_recheck=true"
-            );
-        } else {
-            panic!("Expected a Query leaf");
+    fn test_and_depth(expr: &ScalarIndexExpr) -> usize {
+        match expr {
+            ScalarIndexExpr::And(lhs, rhs) => 1 + test_and_depth(lhs).max(test_and_depth(rhs)),
+            ScalarIndexExpr::Or(lhs, rhs) => test_and_depth(lhs).max(test_and_depth(rhs)),
+            ScalarIndexExpr::Not(inner) => test_and_depth(inner),
+            ScalarIndexExpr::Query(_) => 0,
+        }
+    }
+
+    fn balanced_depth_bound(mut term_count: usize) -> usize {
+        let mut depth = 0;
+        while term_count > 1 {
+            term_count = term_count.div_ceil(2);
+            depth += 1;
         }
+        depth
+    }
+
+    fn int64_index_info(index_type: &str, needs_recheck: bool) -> MockIndexInfoProvider {
+        let parser = |column: &str| {
+            ColInfo::new(
+                DataType::Int64,
+                Box::new(SargableQueryParser::new(
+                    format!("{}_idx", column),
+                    index_type.to_string(),
+                    needs_recheck,
+                )),
+            )
+        };
+        MockIndexInfoProvider::new(vec![("x", parser("x")), ("y", parser("y"))])
+    }
+
+    fn parse_int64_filter(
+        expr: &str,
+        index_info: &dyn IndexInformationProvider,
+    ) -> IndexedExpression {
+        let schema = Schema::new(vec![
+            Field::new("x", DataType::Int64, true),
+            Field::new("y", DataType::Int64, true),
+        ]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+        let expr = state.create_logical_expr(expr, &df_schema).unwrap();
+        apply_scalar_indices(expr, index_info).unwrap()
+    }
+
+    fn optimize_parsed_scalar_filter(
+        expr: &str,
+        index_info: &dyn IndexInformationProvider,
+    ) -> Vec<ScalarIndexExpr> {
+        let indexed = parse_int64_filter(expr, index_info);
+        collect_test_and_leaves(indexed.scalar_query.unwrap().optimize())
+    }
+
+    #[test]
+    fn test_optimize_does_not_remove_is_not_null_for_recheck_range() {
+        let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query(
+            "x",
+            "idx_x",
+            SargableQuery::IsNull(),
+        )));
+        let mut range = test_scalar_range(
+            "x",
+            "idx_x",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+        let ScalarIndexExpr::Query(search) = &mut range else {
+            panic!("expected a range query");
+        };
+        search.needs_recheck = true;
+
+        let leaves = collect_test_and_leaves(test_and_terms(vec![is_not_null, range]).optimize());
+
+        assert_eq!(leaves.len(), 2);
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Not(inner)
+                    if matches!(inner.as_ref(), ScalarIndexExpr::Query(search)
+                        if matches!(search.sargable_query(), Some(SargableQuery::IsNull()))
+                            && !search.needs_recheck)
+            )
+        }));
+    }
+
+    #[test]
+    fn test_optimize_does_not_remove_is_not_null_across_or() {
+        let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query(
+            "x",
+            "idx_x",
+            SargableQuery::IsNull(),
+        )));
+        let range = test_scalar_range(
+            "x",
+            "idx_x",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+        let other = test_scalar_query(
+            "y",
+            "idx_y",
+            SargableQuery::Equals(ScalarValue::Int64(Some(1))),
+        );
+        let disjunction = ScalarIndexExpr::Or(Box::new(range), Box::new(other));
+
+        let leaves = collect_test_and_leaves(
+            ScalarIndexExpr::And(Box::new(is_not_null), Box::new(disjunction)).optimize(),
+        );
+
+        assert_eq!(leaves.len(), 2);
+        assert!(
+            leaves
+                .iter()
+                .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_)))
+        );
+        assert!(
+            leaves
+                .iter()
+                .any(|leaf| matches!(leaf, ScalarIndexExpr::Or(_, _)))
+        );
+    }
+
+    #[test]
+    fn test_optimize_does_not_remove_is_not_null_for_different_index() {
+        let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query(
+            "x",
+            "idx_x",
+            SargableQuery::IsNull(),
+        )));
+        let range = test_scalar_range(
+            "x",
+            "idx_x_other",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+
+        let leaves = collect_test_and_leaves(
+            ScalarIndexExpr::And(Box::new(is_not_null), Box::new(range)).optimize(),
+        );
+
+        assert_eq!(leaves.len(), 2);
+        assert!(
+            leaves
+                .iter()
+                .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_)))
+        );
+    }
+
+    #[test]
+    fn test_optimize_does_not_merge_different_index_types() {
+        let range = |index_type: &str, lower, upper| {
+            ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: "x".to_string(),
+                index_name: "idx_x".to_string(),
+                index_type: index_type.to_string(),
+                query: Arc::new(SargableQuery::Range(lower, upper)),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            })
+        };
+        let btree = range(
+            "BTree",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+        let zone_map = range(
+            "ZoneMap",
+            Bound::Unbounded,
+            Bound::Included(ScalarValue::Int64(Some(20))),
+        );
+
+        let leaves = collect_test_and_leaves(test_and_terms(vec![btree, zone_map]).optimize());
+
+        assert_eq!(leaves.len(), 2);
+    }
+
+    #[test]
+    fn test_optimize_parser_exact_removes_is_not_null_and_merges_ranges() {
+        let index_info = int64_index_info("BTree", false);
+
+        let leaves = optimize_parsed_scalar_filter(
+            "x IS NOT NULL AND y = 1 AND x >= 10 AND x <= 20",
+            &index_info,
+        );
+
+        assert_eq!(leaves.len(), 2);
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Query(search)
+                    if search.column == "x"
+                        && matches!(
+                            search.sargable_query(),
+                            Some(SargableQuery::Range(
+                                Bound::Included(ScalarValue::Int64(Some(10))),
+                                Bound::Included(ScalarValue::Int64(Some(20))),
+                            ))
+                        )
+            )
+        }));
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Query(search)
+                    if search.column == "y"
+                        && matches!(
+                            search.sargable_query(),
+                            Some(SargableQuery::Equals(ScalarValue::Int64(Some(1))))
+                        )
+            )
+        }));
+        assert!(
+            !leaves
+                .iter()
+                .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_)))
+        );
+    }
+
+    #[test]
+    fn test_optimize_parser_preserves_standalone_null_checks() {
+        let index_info = int64_index_info("BTree", false);
+
+        let is_not_null = optimize_parsed_scalar_filter("x IS NOT NULL", &index_info);
+        assert_eq!(is_not_null.len(), 1);
+        assert!(matches!(&is_not_null[0], ScalarIndexExpr::Not(_)));
+
+        let is_null = optimize_parsed_scalar_filter("x IS NULL", &index_info);
+        assert_eq!(is_null.len(), 1);
+        assert!(matches!(
+            &is_null[0],
+            ScalarIndexExpr::Query(search)
+                if matches!(search.sargable_query(), Some(SargableQuery::IsNull()))
+        ));
+    }
+
+    #[test]
+    fn test_optimize_parser_does_not_remove_is_not_null_for_different_column() {
+        let index_info = int64_index_info("BTree", false);
+
+        let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND y >= 10", &index_info);
+
+        assert_eq!(leaves.len(), 2);
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Not(inner)
+                    if matches!(inner.as_ref(), ScalarIndexExpr::Query(search)
+                        if search.column == "x"
+                            && matches!(search.sargable_query(), Some(SargableQuery::IsNull())))
+            )
+        }));
+    }
+
+    #[test]
+    fn test_optimize_parser_handles_in_list_null_semantics() {
+        let index_info = int64_index_info("BTree", false);
+
+        let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x IN (1, 2)", &index_info);
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Query(search)
+                if matches!(search.sargable_query(), Some(SargableQuery::IsIn(values)) if values.len() == 2)
+        ));
+
+        let indexed = parse_int64_filter("x IS NOT NULL AND x IN (1, NULL)", &index_info);
+        assert!(indexed.refine_expr.is_some());
+        let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize());
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(&leaves[0], ScalarIndexExpr::Not(_)));
+    }
+
+    #[test]
+    fn test_optimize_parser_removes_is_not_null_from_not_equal() {
+        let index_info = int64_index_info("BTree", false);
+
+        let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x != 5", &index_info);
+
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Not(inner)
+                if matches!(inner.as_ref(), ScalarIndexExpr::Query(search)
+                    if matches!(search.sargable_query(), Some(SargableQuery::Equals(value))
+                        if *value == ScalarValue::Int64(Some(5))))
+        ));
+    }
+
+    #[test]
+    fn test_optimize_parser_merges_recheck_ranges() {
+        let index_info = int64_index_info("ZoneMap", true);
+
+        let leaves = optimize_parsed_scalar_filter("x >= 10 AND y = 1 AND x <= 20", &index_info);
+
+        assert_eq!(leaves.len(), 2);
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Query(search)
+                    if search.column == "x"
+                        && search.needs_recheck
+                        && matches!(
+                            search.sargable_query(),
+                            Some(SargableQuery::Range(
+                                Bound::Included(ScalarValue::Int64(Some(10))),
+                                Bound::Included(ScalarValue::Int64(Some(20))),
+                            ))
+                        )
+            )
+        }));
+    }
+
+    #[test]
+    fn test_optimize_parser_keeps_recheck_is_not_null_as_refine() {
+        let index_info = int64_index_info("ZoneMap", true);
+
+        let indexed = parse_int64_filter("x IS NOT NULL AND x >= 10", &index_info);
+
+        assert!(indexed.scalar_query.is_some());
+        assert!(matches!(
+            indexed.refine_expr.as_ref(),
+            Some(Expr::IsNotNull(expr))
+                if matches!(expr.as_ref(), Expr::Column(column) if column.name == "x")
+        ));
+        let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize());
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Query(search)
+                if search.needs_recheck
+                    && matches!(search.sargable_query(), Some(SargableQuery::Range(_, _)))
+        ));
+    }
+
+    #[test]
+    fn test_optimize_preserves_balanced_depth_for_unmerged_terms() {
+        let term_count = 2048;
+        let terms = (0..term_count)
+            .map(|value| {
+                test_scalar_query(
+                    "x",
+                    "idx_x",
+                    SargableQuery::Equals(ScalarValue::Int64(Some(value))),
+                )
+            })
+            .collect::<Vec<_>>();
+
+        let optimized = test_and_terms(terms).optimize();
+
+        assert_eq!(
+            collect_test_and_leaves(optimized.clone()).len(),
+            term_count as usize
+        );
+        assert!(
+            test_and_depth(&optimized) <= balanced_depth_bound(term_count as usize),
+            "optimized AND depth should stay balanced, got {} for {} terms",
+            test_and_depth(&optimized),
+            term_count
+        );
+    }
+
+    #[test]
+    fn test_optimize_merges_ranges_inside_not() {
+        let lower = test_scalar_range(
+            "x",
+            "idx_x",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+        let upper = test_scalar_range(
+            "x",
+            "idx_x",
+            Bound::Unbounded,
+            Bound::Included(ScalarValue::Int64(Some(20))),
+        );
+        let sibling = test_scalar_query(
+            "y",
+            "idx_y",
+            SargableQuery::Equals(ScalarValue::Int64(Some(1))),
+        );
+
+        let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![lower, upper])));
+        let optimized = ScalarIndexExpr::And(Box::new(nested_not), Box::new(sibling)).optimize();
+
+        let ScalarIndexExpr::And(nested_not, _) = optimized else {
+            panic!("expected outer AND expression");
+        };
+        let ScalarIndexExpr::Not(inner) = *nested_not else {
+            panic!("expected NOT expression");
+        };
+        let leaves = collect_test_and_leaves(*inner);
+
+        assert_eq!(leaves.len(), 1);
+        assert!(matches!(
+            &leaves[0],
+            ScalarIndexExpr::Query(search)
+                if matches!(
+                    search.sargable_query(),
+                    Some(SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(10))),
+                        Bound::Included(ScalarValue::Int64(Some(20))),
+                    ))
+                )
+        ));
+    }
+
+    #[test]
+    fn test_optimize_does_not_remove_is_not_null_inside_not() {
+        let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query(
+            "x",
+            "idx_x",
+            SargableQuery::IsNull(),
+        )));
+        let range = test_scalar_range(
+            "x",
+            "idx_x",
+            Bound::Included(ScalarValue::Int64(Some(10))),
+            Bound::Unbounded,
+        );
+        let guarded_range = test_and_terms(vec![is_not_null, range]);
+        let alternative = test_scalar_query(
+            "y",
+            "idx_y",
+            SargableQuery::Equals(ScalarValue::Int64(Some(1))),
+        );
+        let or = ScalarIndexExpr::Or(Box::new(guarded_range), Box::new(alternative));
+        let sibling = test_scalar_query(
+            "z",
+            "idx_z",
+            SargableQuery::Equals(ScalarValue::Int64(Some(2))),
+        );
+        let outer_sibling = test_scalar_query(
+            "w",
+            "idx_w",
+            SargableQuery::Equals(ScalarValue::Int64(Some(3))),
+        );
+
+        let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![or, sibling])));
+        let optimized =
+            ScalarIndexExpr::And(Box::new(nested_not), Box::new(outer_sibling)).optimize();
+
+        let ScalarIndexExpr::And(nested_not, _) = optimized else {
+            panic!("expected outer AND expression");
+        };
+        let ScalarIndexExpr::Not(inner) = *nested_not else {
+            panic!("expected NOT expression");
+        };
+        let ScalarIndexExpr::And(or, _) = *inner else {
+            panic!("expected AND expression");
+        };
+        let ScalarIndexExpr::Or(lhs, _) = *or else {
+            panic!("expected OR expression");
+        };
+        let leaves = collect_test_and_leaves(*lhs);
+
+        assert_eq!(leaves.len(), 2);
+        assert!(leaves.iter().any(|leaf| {
+            matches!(
+                leaf,
+                ScalarIndexExpr::Not(inner)
+                    if matches!(inner.as_ref(), ScalarIndexExpr::Query(search)
+                        if matches!(search.sargable_query(), Some(SargableQuery::IsNull())))
+            )
+        }));
     }
 
     #[test]
@@ -4268,7 +5103,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(leaves.len(), 1, "All 4 ranges should merge into 1");
 
             if let ScalarIndexExpr::Query(s) = &leaves[0] {
@@ -4305,7 +5140,7 @@ mod tests {
             let optimized = expr.optimize();
 
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(
                 leaves.len(),
                 2,
@@ -4353,7 +5188,7 @@ mod tests {
             if let ScalarIndexExpr::Or(lhs, rhs) = &optimized {
                 // Each branch should be a single merged range
                 let mut left_leaves = Vec::new();
-                lhs.clone().collect_and_leaves(&mut left_leaves);
+                lhs.clone().collect_and_leaves(&mut left_leaves, true);
                 assert_eq!(
                     left_leaves.len(),
                     1,
@@ -4371,7 +5206,7 @@ mod tests {
                 }
 
                 let mut right_leaves = Vec::new();
-                rhs.clone().collect_and_leaves(&mut right_leaves);
+                rhs.clone().collect_and_leaves(&mut right_leaves, true);
                 assert_eq!(
                     right_leaves.len(),
                     1,
@@ -4429,7 +5264,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(
                 leaves.len(),
                 2,
@@ -4502,7 +5337,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(leaves.len(), 3, "fqdn + merged_time + channel = 3 leaves");
 
             let time_leaf = leaves
@@ -4541,7 +5376,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(leaves.len(), 1);
             if let ScalarIndexExpr::Query(s) = &leaves[0] {
                 let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
@@ -4575,7 +5410,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(leaves.len(), 1);
             if let ScalarIndexExpr::Query(s) = &leaves[0] {
                 let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
@@ -4610,7 +5445,7 @@ mod tests {
 
             let optimized = expr.optimize();
             let mut leaves = Vec::new();
-            optimized.collect_and_leaves(&mut leaves);
+            optimized.collect_and_leaves(&mut leaves, true);
             assert_eq!(leaves.len(), 1);
             if let ScalarIndexExpr::Query(s) = &leaves[0] {
                 let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs
index 7b87e492ec2..c79e4a33274 100644
--- a/rust/lance-index/src/scalar/fmindex.rs
+++ b/rust/lance-index/src/scalar/fmindex.rs
@@ -1701,7 +1701,7 @@ impl FMIndexScalarIndex {
         }
         pfiles.sort_by_key(|(id, _)| *id);
         let io_parallelism = store.io_parallelism().max(1);
-        let mut parts = futures::stream::iter(pfiles.into_iter())
+        let mut parts = futures::stream::iter(pfiles)
             .map(|(id, name)| {
                 let store = Arc::clone(&store);
                 async move {
diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs
index 6c9a5ad2947..72604243c1c 100644
--- a/rust/lance-index/src/scalar/inverted.rs
+++ b/rust/lance-index/src/scalar/inverted.rs
@@ -4,6 +4,7 @@
 pub mod builder;
 mod cache_codec;
 mod encoding;
+mod impact;
 mod index;
 mod iter;
 pub mod json;
@@ -91,7 +92,7 @@ pub async fn build_global_bm25_scorer(
     let (mut total_tokens, mut num_docs, first_token_docs) =
         first_index.bm25_stats_for_terms(&terms).await?;
     let mut token_docs = HashMap::with_capacity(terms.len());
-    for (term, count) in terms.iter().cloned().zip(first_token_docs.into_iter()) {
+    for (term, count) in terms.iter().cloned().zip(first_token_docs) {
         token_docs.insert(term, count);
     }
 
@@ -100,7 +101,7 @@ pub async fn build_global_bm25_scorer(
             index.bm25_stats_for_terms(&terms).await?;
         total_tokens += segment_total_tokens;
         num_docs += segment_num_docs;
-        for (term, count) in terms.iter().zip(segment_token_docs.into_iter()) {
+        for (term, count) in terms.iter().zip(segment_token_docs) {
             *token_docs
                 .get_mut(term)
                 .expect("global scorer terms should already be initialized") += count;
@@ -146,6 +147,7 @@ impl InvertedIndexPlugin {
             }
         });
 
+        params.validate_format_version()?;
         let format_version = params.resolved_format_version();
         let details = pbold::InvertedIndexDetails::try_from(&params)?;
         let mut inverted_index =
@@ -211,7 +213,7 @@ impl BasicTrainer for InvertedIndexPlugin {
                 .into()))
         }
 
-        let params = serde_json::from_str::<InvertedIndexParams>(params)?;
+        let params = InvertedIndexParams::from_training_json(params)?;
         Ok(Box::new(InvertedIndexTrainingRequest::new(params)))
     }
 
@@ -266,7 +268,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
     }
 
     fn version(&self) -> u32 {
-        max_supported_fts_format_version().index_version()
+        INVERTED_INDEX_VERSION_V3
     }
 
     fn new_query_parser(
@@ -315,13 +317,43 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::scalar::{BuiltinIndexType, ScalarIndexParams};
 
     #[test]
-    fn test_plugin_version_tracks_max_supported_format() {
+    fn test_plugin_version_tracks_v3_capability_gate() {
         let plugin = InvertedIndexPlugin;
-        assert_eq!(
-            plugin.version(),
-            max_supported_fts_format_version().index_version()
-        );
+        assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3);
+    }
+
+    #[test]
+    fn test_new_training_request_defaults_missing_block_size_to_128() {
+        let plugin = InvertedIndexPlugin;
+        let field = Field::new("text", DataType::Utf8, true);
+
+        let cases = [
+            (
+                ScalarIndexParams::for_builtin(BuiltinIndexType::Inverted),
+                false,
+            ),
+            (ScalarIndexParams::new("inverted".to_string()), false),
+            (
+                ScalarIndexParams::new("inverted".to_string())
+                    .with_params(&serde_json::json!({ "with_position": true })),
+                true,
+            ),
+        ];
+
+        for (params, expected_with_position) in cases {
+            let request = plugin
+                .new_training_request(params.params.as_deref().unwrap_or("{}"), &field)
+                .unwrap();
+            let request = request
+                .as_any()
+                .downcast_ref::<InvertedIndexTrainingRequest>()
+                .unwrap();
+
+            assert_eq!(request.parameters.posting_block_size(), DEFAULT_BLOCK_SIZE);
+            assert_eq!(request.parameters.has_positions(), expected_with_position);
+        }
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs
index ede00ad43ee..c90f96053ef 100644
--- a/rust/lance-index/src/scalar/inverted/builder.rs
+++ b/rust/lance-index/src/scalar/inverted/builder.rs
@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use super::encoding::encode_group_starts;
 use super::{InvertedIndexParams, index::*};
 use crate::scalar::inverted::document_tokenizer::DocType;
 use crate::scalar::inverted::json::JsonTextStream;
+use crate::scalar::inverted::tokenizer::LEGACY_BLOCK_SIZE;
 use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
 #[cfg(test)]
 use crate::scalar::lance_format::LanceIndexStore;
@@ -13,12 +13,11 @@ use crate::vector::graph::OrderedFloat;
 use crate::{progress::IndexBuildProgress, progress::noop_progress};
 use arrow::array::AsArray;
 use arrow::datatypes;
-use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array};
+use arrow_array::{Array, BinaryArray, RecordBatch};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
-use bytes::Bytes;
-use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream};
+use datafusion::execution::SendableRecordBatchStream;
 use fst::Streamer;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use lance_arrow::json::JSON_EXT_NAME;
 use lance_arrow::{ARROW_EXT_NAME_KEY, iter_str_array};
 use lance_bitpacking::{BitPacker, BitPacker4x};
@@ -27,24 +26,22 @@ use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::LanceOptionExt;
 use lance_core::utils::row_addr_remap::RowAddrRemap;
 use lance_core::utils::tokio::{IO_CORE_RESERVATION, get_num_compute_intensive_cpus, spawn_cpu};
-use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result};
+use lance_core::{Error, ROW_ID, Result};
 use lance_io::object_store::ObjectStore;
 use lance_select::RowSetOps;
 use object_store::path::Path;
 use roaring::RoaringBitmap;
 use smallvec::SmallVec;
 use std::collections::HashMap;
-use std::pin::Pin;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::sync::LazyLock;
-use std::task::{Context, Poll};
 use std::{fmt::Debug, sync::atomic::AtomicU64};
 use tracing::instrument;
 
-// the number of elements in each block
-// each block contains 128 row ids and 128 frequencies
-// WARNING: changing this value will break the compatibility with existing indexes
+// The legacy bitpacking block size. Position streams still use this block size;
+// FTS posting blocks choose their physical bitpacker from the configured
+// InvertedIndexParams::block_size.
 pub const BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 
 // The default number of workers to use for FTS builds.
@@ -75,92 +72,8 @@ static LANCE_FTS_POSTING_BATCH_ROWS: LazyLock<usize> = LazyLock::new(|| {
         .parse()
         .expect("failed to parse LANCE_FTS_POSTING_BATCH_ROWS")
 });
-// Target serialized byte size of a posting-list cache group. Consecutive
-// posting lists are grouped into a single cache entry until their combined
-// serialized size reaches this target, amortizing per-entry overhead across
-// small (Zipfian-rare) terms. See issue #7040.
-static LANCE_FTS_POSTING_GROUP_TARGET_BYTES: LazyLock<usize> = LazyLock::new(|| {
-    std::env::var("LANCE_FTS_POSTING_GROUP_TARGET_BYTES")
-        .unwrap_or_else(|_| "4096".to_string())
-        .parse()
-        .expect("failed to parse LANCE_FTS_POSTING_GROUP_TARGET_BYTES")
-});
-// Maximum number of posting lists in a single cache group, regardless of byte
-// size. Caps the work and memory of a single group read for corpora with many
-// tiny terms.
-static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock<usize> = LazyLock::new(|| {
-    std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
-        .unwrap_or_else(|_| "256".to_string())
-        .parse()
-        .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
-});
 const MAX_RETAINED_TOKEN_IDS: usize = 8 * 1024;
 
-/// Write-time configuration controlling how consecutive posting lists are
-/// grouped into a single read-path cache entry (issue #7040). Defaults come
-/// from the `LANCE_FTS_POSTING_GROUP_*` environment variables.
-#[derive(Debug, Clone, Copy)]
-pub(crate) struct PostingGroupConfig {
-    pub(crate) target_bytes: usize,
-    pub(crate) max_tokens: usize,
-}
-
-impl Default for PostingGroupConfig {
-    fn default() -> Self {
-        Self {
-            target_bytes: (*LANCE_FTS_POSTING_GROUP_TARGET_BYTES).max(1),
-            max_tokens: (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1),
-        }
-    }
-}
-
-/// Accumulates posting-list group boundaries at write time. Tokens are pushed
-/// in row order; a group is cut once its serialized bytes reach
-/// `target_bytes` or it holds `max_tokens` posting lists. A posting list
-/// larger than the target that *starts* a group occupies that group alone (the
-/// clamp case); one encountered mid-group is absorbed and closes that group, so
-/// a single term is never split across groups.
-#[derive(Debug)]
-pub(crate) struct PostingGroupAccumulator {
-    config: PostingGroupConfig,
-    starts: Vec<u32>,
-    next_token: u32,
-    current_bytes: usize,
-    current_tokens: usize,
-}
-
-impl PostingGroupAccumulator {
-    pub(crate) fn new(config: PostingGroupConfig) -> Self {
-        Self {
-            config,
-            starts: Vec::new(),
-            next_token: 0,
-            current_bytes: 0,
-            current_tokens: 0,
-        }
-    }
-
-    /// Record the next posting list in row order, given its serialized byte size.
-    pub(crate) fn push(&mut self, posting_bytes: usize) {
-        if self.current_tokens == 0 {
-            self.starts.push(self.next_token);
-        }
-        self.current_bytes += posting_bytes;
-        self.current_tokens += 1;
-        self.next_token += 1;
-        if self.current_bytes >= self.config.target_bytes
-            || self.current_tokens >= self.config.max_tokens
-        {
-            self.current_bytes = 0;
-            self.current_tokens = 0;
-        }
-    }
-
-    pub(crate) fn into_starts(self) -> Vec<u32> {
-        self.starts
-    }
-}
-
 fn default_num_workers() -> usize {
     let total_cpus = get_num_compute_intensive_cpus() + *IO_CORE_RESERVATION;
     std::cmp::max(1, total_cpus / 2)
@@ -282,8 +195,11 @@ impl InvertedIndexBuilder {
     }
 
     pub fn with_posting_tail_codec(mut self, posting_tail_codec: PostingTailCodec) -> Self {
-        self.format_version =
-            InvertedListFormatVersion::from_posting_tail_codec(posting_tail_codec);
+        self.format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            posting_tail_codec,
+            self.params.block_size,
+        )
+        .expect("invalid posting tail codec for posting block size");
         self.posting_tail_codec = posting_tail_codec;
         self
     }
@@ -310,6 +226,7 @@ impl InvertedIndexBuilder {
         dest_store: &dyn IndexStore,
         old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
     ) -> Result<Vec<IndexFile>> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let schema = new_data.schema();
         let doc_col = schema.field(0).name();
 
@@ -344,6 +261,7 @@ impl InvertedIndexBuilder {
         old_segments: &[Arc<InvertedIndex>],
         old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
     ) -> Result<Vec<IndexFile>> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let schema = new_data.schema();
         let doc_col = schema.field(0).name();
 
@@ -469,6 +387,7 @@ impl InvertedIndexBuilder {
             fragment_mask: self.fragment_mask,
             token_set_format: self.token_set_format,
             worker_memory_limit_bytes,
+            block_size: self.params.block_size,
         };
         let next_id = self.next_partition_id();
         let id_alloc = Arc::new(AtomicU64::new(next_id));
@@ -619,6 +538,7 @@ impl InvertedIndexBuilder {
         dest_store: &dyn IndexStore,
         partitions: &[u64],
     ) -> Result<IndexFile> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let mut serialized_deleted_fragments =
             Vec::with_capacity(self.deleted_fragments.serialized_size());
         self.deleted_fragments
@@ -635,6 +555,14 @@ impl InvertedIndexBuilder {
                 POSTING_TAIL_CODEC_KEY.to_owned(),
                 self.posting_tail_codec.as_str().to_owned(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                self.format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                self.params.block_size.to_string(),
+            ),
         ]);
 
         if self.params.with_position && self.format_version.uses_shared_position_stream() {
@@ -679,6 +607,7 @@ impl InvertedIndexBuilder {
         dest_store: &dyn IndexStore,
         partition: u64, // Modify parameter type
     ) -> Result<IndexFile> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let partitions = vec![partition];
         let mut metadata = HashMap::from_iter(vec![
             ("partitions".to_owned(), serde_json::to_string(&partitions)?),
@@ -691,6 +620,14 @@ impl InvertedIndexBuilder {
                 POSTING_TAIL_CODEC_KEY.to_owned(),
                 self.posting_tail_codec.as_str().to_owned(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                self.format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                self.params.block_size.to_string(),
+            ),
         ]);
         if self.params.with_position && self.format_version.uses_shared_position_stream() {
             metadata.insert(
@@ -827,10 +764,10 @@ pub struct InnerBuilder {
     token_set_format: TokenSetFormat,
     format_version: InvertedListFormatVersion,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     pub(crate) tokens: TokenSet,
     pub(crate) posting_lists: Vec<PostingListBuilder>,
     pub(crate) docs: DocSet,
-    pub(crate) group_config: PostingGroupConfig,
 }
 
 impl InnerBuilder {
@@ -849,16 +786,51 @@ impl InnerBuilder {
         token_set_format: TokenSetFormat,
         format_version: InvertedListFormatVersion,
     ) -> Self {
+        Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        block_size: usize,
+    ) -> Self {
+        let format_version = default_fts_format_version_for_block_size(block_size)
+            .expect("invalid posting list block size");
+        Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            block_size,
+        )
+    }
+
+    pub fn new_with_format_version_and_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        format_version: InvertedListFormatVersion,
+        block_size: usize,
+    ) -> Self {
+        validate_format_version_block_size(format_version, block_size)
+            .expect("invalid FTS format version for posting block size");
         Self {
             id,
             with_position,
             token_set_format,
             format_version,
             posting_tail_codec: format_version.posting_tail_codec(),
+            block_size,
             tokens: TokenSet::default(),
             posting_lists: Vec::new(),
             docs: DocSet::default(),
-            group_config: PostingGroupConfig::default(),
         }
     }
 
@@ -868,13 +840,34 @@ impl InnerBuilder {
         token_set_format: TokenSetFormat,
         posting_tail_codec: PostingTailCodec,
     ) -> Self {
-        let format_version = if posting_tail_codec == PostingTailCodec::Fixed32 {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
-        let mut builder =
-            Self::new_with_format_version(id, with_position, token_set_format, format_version);
+        Self::new_with_posting_tail_codec_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            posting_tail_codec,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_posting_tail_codec_and_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Self {
+        let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            posting_tail_codec,
+            block_size,
+        )
+        .expect("invalid posting tail codec for posting block size");
+        let mut builder = Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            block_size,
+        );
         builder.posting_tail_codec = posting_tail_codec;
         builder
     }
@@ -955,10 +948,10 @@ impl InnerBuilder {
             token_set_format,
             format_version,
             posting_tail_codec,
+            block_size,
             tokens,
             posting_lists,
             docs,
-            group_config: _,
         } = other;
 
         if self.with_position != with_position {
@@ -985,6 +978,12 @@ impl InnerBuilder {
                 self.posting_tail_codec, posting_tail_codec
             )));
         }
+        if self.block_size != block_size {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched FTS block sizes: {} vs {}",
+                self.block_size, block_size
+            )));
+        }
 
         let mut token_id_map = vec![u32::MAX; posting_lists.len()];
         match tokens.tokens {
@@ -1010,7 +1009,11 @@ impl InnerBuilder {
             self.docs.append(*row_id, *num_tokens);
         }
         self.posting_lists.resize_with(self.tokens.len(), || {
-            PostingListBuilder::new_with_posting_tail_codec(with_position, self.posting_tail_codec)
+            PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                with_position,
+                self.posting_tail_codec,
+                self.block_size,
+            )
         });
 
         for (token_id, posting_list) in posting_lists.into_iter().enumerate() {
@@ -1077,7 +1080,11 @@ impl InnerBuilder {
         let mut writer = store
             .new_index_file(
                 path,
-                inverted_list_schema_for_version(self.with_position, self.format_version),
+                inverted_list_schema_for_version_with_block_size(
+                    self.with_position,
+                    self.format_version,
+                    self.block_size,
+                ),
             )
             .await?;
         let posting_lists = std::mem::take(&mut self.posting_lists);
@@ -1090,8 +1097,11 @@ impl InnerBuilder {
         );
         let with_position = self.with_position;
         let format_version = self.format_version;
-        let group_config = self.group_config;
-        let schema = inverted_list_schema_for_version(self.with_position, self.format_version);
+        let schema = inverted_list_schema_for_version_with_block_size(
+            self.with_position,
+            self.format_version,
+            self.block_size,
+        );
         let docs_for_batches = docs.clone();
         let schema_for_batches = schema.clone();
         let batch_rows = *LANCE_FTS_POSTING_BATCH_ROWS;
@@ -1111,15 +1121,13 @@ impl InnerBuilder {
                 with_position,
                 format_version,
                 batch_rows,
-                group_config,
             );
             let mut posting_lists = posting_lists.into_iter();
             loop {
                 let docs_for_batches = docs_for_batches.clone();
                 // Build the next batch on the CPU pool. The builder and the
                 // remaining posting lists are moved in and handed back so state
-                // persists across batches -- notably the cache-group accumulator,
-                // which spans every batch this builder produces.
+                // persists across batches.
                 let (next_builder, next_posting_lists, batch) = spawn_cpu(move || {
                     let mut batch_builder = batch_builder;
                     let mut posting_lists = posting_lists;
@@ -1155,7 +1163,7 @@ impl InnerBuilder {
                 }
             }
 
-            Result::Ok(batch_builder.into_group_starts())
+            Result::Ok(())
         });
 
         while let Ok(batch) = rx.recv().await {
@@ -1167,22 +1175,8 @@ impl InnerBuilder {
             }
         }
         drop(rx);
-        let group_starts = producer.await??;
-
-        // Persist the posting-list cache-group boundaries as a global buffer,
-        // recording its 1-indexed id in schema metadata so the reader can group
-        // small posting lists into a single cache entry (issue #7040). Empty
-        // partitions skip this entirely and fall back to the per-token path.
-        let mut extra_metadata = HashMap::new();
-        if !group_starts.is_empty() {
-            let encoded = encode_group_starts(&group_starts);
-            let buffer_id = writer.add_global_buffer(Bytes::from(encoded)).await?;
-            extra_metadata.insert(
-                POSTING_GROUP_OFFSETS_BUF_KEY.to_owned(),
-                buffer_id.to_string(),
-            );
-        }
-        writer.finish_with_metadata(extra_metadata).await
+        producer.await??;
+        writer.finish().await
     }
 
     #[instrument(level = "debug", skip_all)]
@@ -1264,6 +1258,11 @@ struct WorkerOutput {
     tail_partition: Option<TailPartition>,
 }
 
+enum DocumentSource<'a> {
+    Text(&'a str),
+    StringList(&'a dyn Array),
+}
+
 #[derive(Debug, Clone, Copy)]
 struct IndexWorkerConfig {
     with_position: bool,
@@ -1271,6 +1270,7 @@ struct IndexWorkerConfig {
     fragment_mask: Option<u64>,
     token_set_format: TokenSetFormat,
     worker_memory_limit_bytes: u64,
+    block_size: usize,
 }
 
 impl IndexWorker {
@@ -1314,17 +1314,22 @@ impl IndexWorker {
         id_alloc: Arc<AtomicU64>,
         config: IndexWorkerConfig,
     ) -> Result<Self> {
-        let schema = inverted_list_schema_for_version(config.with_position, config.format_version);
+        let schema = inverted_list_schema_for_version_with_block_size(
+            config.with_position,
+            config.format_version,
+            config.block_size,
+        );
 
         Ok(Self {
             tokenizer,
             dest_store,
-            builder: InnerBuilder::new_with_format_version(
+            builder: InnerBuilder::new_with_format_version_and_block_size(
                 id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
                     | config.fragment_mask.unwrap_or(0),
                 config.with_position,
                 config.token_set_format,
                 config.format_version,
+                config.block_size,
             ),
             partitions: Vec::new(),
             files: Vec::new(),
@@ -1349,147 +1354,254 @@ impl IndexWorker {
 
     async fn process_batch(&mut self, batch: RecordBatch) -> Result<()> {
         let doc_col = batch.column(0);
-        let doc_iter = iter_str_array(doc_col);
         let row_id_col = batch[ROW_ID].as_primitive::<datatypes::UInt64Type>();
-        let docs = doc_iter
-            .zip(row_id_col.values().iter())
-            .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id)));
+        match doc_col.data_type() {
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                let docs = iter_str_array(doc_col.as_ref())
+                    .zip(row_id_col.values().iter())
+                    .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id)));
+
+                for (doc, row_id) in docs {
+                    self.process_document(row_id, DocumentSource::Text(doc))
+                        .await?;
+                }
+            }
+            DataType::List(_) => {
+                self.process_string_list_batch::<i32>(doc_col, row_id_col)
+                    .await?;
+            }
+            DataType::LargeList(_) => {
+                self.process_string_list_batch::<i64>(doc_col, row_id_col)
+                    .await?;
+            }
+            data_type => {
+                return Err(Error::index(format!(
+                    "expect data type String, LargeString, List(String), or LargeList(String) but got {}",
+                    data_type
+                )));
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn process_string_list_batch<Offset: arrow::array::OffsetSizeTrait>(
+        &mut self,
+        doc_col: &Arc<dyn Array>,
+        row_id_col: &arrow_array::PrimitiveArray<datatypes::UInt64Type>,
+    ) -> Result<()> {
+        let docs = doc_col.as_list::<Offset>();
+        match docs.value_type() {
+            datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => {}
+            data_type => {
+                return Err(Error::index(format!(
+                    "expect list item data type String or LargeString but got {}",
+                    data_type
+                )));
+            }
+        }
+
+        for (doc, row_id) in docs.iter().zip(row_id_col.values().iter()) {
+            let Some(doc) = doc else {
+                continue;
+            };
+
+            self.process_document(*row_id, DocumentSource::StringList(doc.as_ref()))
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    fn checked_token_position(row_id: u64, token_position: usize) -> Result<u32> {
+        u32::try_from(token_position).map_err(|_| {
+            Error::invalid_input(format!(
+                "token position overflow for row_id={row_id}: token_position={token_position}"
+            ))
+        })
+    }
 
+    fn materialize_string_list(elements: &dyn Array) -> String {
+        let mut doc = String::new();
+        for element in iter_str_array(elements).flatten() {
+            if !doc.is_empty() {
+                doc.push(' ');
+            }
+            doc.push_str(element);
+        }
+        doc
+    }
+
+    async fn process_document(&mut self, row_id: u64, document: DocumentSource<'_>) -> Result<()> {
         let with_position = self.has_position();
-        for (doc, row_id) in docs {
-            let builder_was_empty = self.builder.docs.is_empty();
-            let old_temporary_memory_size = self.temporary_memory_size();
-            let old_token_memory_size = self.builder.tokens.memory_size() as u64;
-            let doc_id = self.builder.docs.len() as u32;
-            let mut token_num: u32 = 0;
-            let mut posting_memory_delta = 0i64;
-            if with_position {
+        let builder_was_empty = self.builder.docs.is_empty();
+        let old_temporary_memory_size = self.temporary_memory_size();
+        let old_token_memory_size = self.builder.tokens.memory_size() as u64;
+        let doc_id = self.builder.docs.len() as u32;
+        let mut token_num: u32 = 0;
+        let mut doc_length_bytes = 0usize;
+        let mut posting_memory_delta = 0i64;
+        if with_position {
+            {
                 if self.token_ids.capacity() < self.last_token_count {
                     self.token_ids
                         .reserve(self.last_token_count - self.token_ids.capacity());
                 }
                 self.token_ids.clear();
+                let tokenizer = &mut self.tokenizer;
                 let builder = &mut self.builder;
                 let token_ids = &mut self.token_ids;
                 let memory_size = &mut self.memory_size;
                 let posting_tail_codec = builder.posting_tail_codec;
 
-                let mut token_stream = self.tokenizer.token_stream_for_doc(doc);
-                while token_stream.advance() {
-                    let token = token_stream.token();
-                    let token_id = builder.tokens.get_or_add(&token.text);
-                    if token_id as usize == builder.posting_lists.len() {
-                        let old_posting_lists_overhead_size = (builder.posting_lists.capacity()
-                            * std::mem::size_of::<PostingListBuilder>())
-                            as u64;
-                        builder.posting_lists.push(
-                            PostingListBuilder::new_with_posting_tail_codec(
-                                true,
-                                posting_tail_codec,
-                            ),
-                        );
-                        let new_posting_lists_overhead_size = (builder.posting_lists.capacity()
-                            * std::mem::size_of::<PostingListBuilder>())
-                            as u64;
-                        Self::adjust_tracked_value(
-                            memory_size,
-                            old_posting_lists_overhead_size,
-                            new_posting_lists_overhead_size,
-                        );
+                let block_size = builder.block_size;
+                let mut process_text = |text: &str| -> Result<()> {
+                    doc_length_bytes += text.len();
+                    let mut token_stream = tokenizer.token_stream_for_doc(text);
+                    while token_stream.advance() {
+                        let token = token_stream.token();
+                        let position = Self::checked_token_position(row_id, token.position)?;
+                        let token_id = builder.tokens.get_or_add(&token.text);
+                        if token_id as usize == builder.posting_lists.len() {
+                            let old_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                                * std::mem::size_of::<PostingListBuilder>())
+                                as u64;
+                            builder.posting_lists.push(
+                                PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                                    true,
+                                    posting_tail_codec,
+                                    block_size,
+                                ),
+                            );
+                            let new_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                                * std::mem::size_of::<PostingListBuilder>())
+                                as u64;
+                            Self::adjust_tracked_value(
+                                memory_size,
+                                old_posting_lists_overhead_size,
+                                new_posting_lists_overhead_size,
+                            );
+                        }
+                        let posting_list = &mut builder.posting_lists[token_id as usize];
+                        let old_posting_memory_size = posting_list.size();
+                        if posting_list.add_occurrence(doc_id, position)? {
+                            token_ids.push(token_id);
+                        }
+                        let new_posting_memory_size = posting_list.size();
+                        posting_memory_delta +=
+                            new_posting_memory_size as i64 - old_posting_memory_size as i64;
+                        token_num += 1;
                     }
-                    let posting_list = &mut builder.posting_lists[token_id as usize];
-                    let old_posting_memory_size = posting_list.size();
-                    if posting_list.add_occurrence(doc_id, token.position as u32)? {
-                        token_ids.push(token_id);
+                    Ok(())
+                };
+
+                match document {
+                    DocumentSource::Text(doc) => {
+                        process_text(doc)?;
+                    }
+                    DocumentSource::StringList(elements) => {
+                        let doc = Self::materialize_string_list(elements);
+                        process_text(&doc)?;
                     }
-                    let new_posting_memory_size = posting_list.size();
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                    token_num += 1;
                 }
-            } else {
+            }
+        } else {
+            {
                 if self.token_ids.capacity() < self.last_token_count {
                     self.token_ids
                         .reserve(self.last_token_count - self.token_ids.capacity());
                 }
                 self.token_ids.clear();
 
-                let mut token_stream = self.tokenizer.token_stream_for_doc(doc);
-                while token_stream.advance() {
-                    let token_id = self.builder.tokens.get_or_add(&token_stream.token().text);
-                    self.token_ids.push(token_id);
-                    token_num += 1;
-                }
-            }
-            self.adjust_tracked_memory_size(
-                old_token_memory_size,
-                self.builder.tokens.memory_size() as u64,
-            );
+                let tokenizer = &mut self.tokenizer;
+                let builder = &mut self.builder;
+                let token_ids = &mut self.token_ids;
+                let mut process_text = |text: &str| {
+                    doc_length_bytes += text.len();
+                    let mut token_stream = tokenizer.token_stream_for_doc(text);
+                    while token_stream.advance() {
+                        let token_id = builder.tokens.get_or_add(&token_stream.token().text);
+                        token_ids.push(token_id);
+                        token_num += 1;
+                    }
+                };
 
-            if !with_position {
-                let old_posting_lists_overhead_size = self.posting_lists_overhead_size();
-                self.builder
-                    .posting_lists
-                    .resize_with(self.builder.tokens.len(), || {
-                        PostingListBuilder::new_with_posting_tail_codec(
-                            false,
-                            self.builder.posting_tail_codec,
-                        )
-                    });
-                let new_posting_lists_overhead_size = self.posting_lists_overhead_size();
-                Self::adjust_tracked_value(
-                    &mut self.memory_size,
-                    old_posting_lists_overhead_size,
-                    new_posting_lists_overhead_size,
-                );
+                match document {
+                    DocumentSource::Text(doc) => process_text(doc),
+                    DocumentSource::StringList(elements) => {
+                        let doc = Self::materialize_string_list(elements);
+                        process_text(&doc);
+                    }
+                }
             }
+        }
+        self.adjust_tracked_memory_size(
+            old_token_memory_size,
+            self.builder.tokens.memory_size() as u64,
+        );
 
-            let old_doc_memory_size = self.builder.docs.memory_size() as u64;
-            let appended_doc_id = self.builder.docs.append(row_id, token_num);
-            debug_assert_eq!(appended_doc_id, doc_id);
+        if token_num == 0 {
+            self.last_token_count = 0;
+            self.trim_temporary_buffers();
             self.adjust_tracked_memory_size(
-                old_doc_memory_size,
-                self.builder.docs.memory_size() as u64,
+                old_temporary_memory_size,
+                self.temporary_memory_size(),
             );
-            self.total_doc_length += doc.len();
+            return Ok(());
+        }
 
-            if with_position {
-                for &token_id in &self.token_ids {
-                    let (old_posting_memory_size, new_posting_memory_size) = {
-                        let posting_list = &mut self.builder.posting_lists[token_id as usize];
-                        let old_posting_memory_size = posting_list.size();
-                        posting_list.finish_open_doc(doc_id)?;
-                        let new_posting_memory_size = posting_list.size();
-                        (old_posting_memory_size, new_posting_memory_size)
-                    };
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                }
-                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
-            } else if token_num > 0 {
-                self.token_ids.sort_unstable();
-                let mut iter = self.token_ids.iter();
-                let mut current = *iter.next().unwrap();
-                let mut count = 1u32;
-                for &token_id in iter {
-                    if token_id == current {
-                        count += 1;
-                        continue;
-                    }
+        if !with_position {
+            let old_posting_lists_overhead_size = self.posting_lists_overhead_size();
+            self.builder
+                .posting_lists
+                .resize_with(self.builder.tokens.len(), || {
+                    PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                        false,
+                        self.builder.posting_tail_codec,
+                        self.builder.block_size,
+                    )
+                });
+            let new_posting_lists_overhead_size = self.posting_lists_overhead_size();
+            Self::adjust_tracked_value(
+                &mut self.memory_size,
+                old_posting_lists_overhead_size,
+                new_posting_lists_overhead_size,
+            );
+        }
 
-                    let (old_posting_memory_size, new_posting_memory_size) = {
-                        let posting_list = &mut self.builder.posting_lists[current as usize];
-                        let old_posting_memory_size = posting_list.size();
-                        posting_list.add(doc_id, PositionRecorder::Count(count));
-                        let new_posting_memory_size = posting_list.size();
-                        (old_posting_memory_size, new_posting_memory_size)
-                    };
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
+        let old_doc_memory_size = self.builder.docs.memory_size() as u64;
+        let appended_doc_id = self.builder.docs.append(row_id, token_num);
+        debug_assert_eq!(appended_doc_id, doc_id);
+        self.adjust_tracked_memory_size(
+            old_doc_memory_size,
+            self.builder.docs.memory_size() as u64,
+        );
+        self.total_doc_length += doc_length_bytes;
 
-                    current = token_id;
-                    count = 1;
+        if with_position {
+            for &token_id in &self.token_ids {
+                let (old_posting_memory_size, new_posting_memory_size) = {
+                    let posting_list = &mut self.builder.posting_lists[token_id as usize];
+                    let old_posting_memory_size = posting_list.size();
+                    posting_list.finish_open_doc(doc_id)?;
+                    let new_posting_memory_size = posting_list.size();
+                    (old_posting_memory_size, new_posting_memory_size)
+                };
+                posting_memory_delta +=
+                    new_posting_memory_size as i64 - old_posting_memory_size as i64;
+            }
+            Self::apply_delta(&mut self.memory_size, posting_memory_delta);
+        } else {
+            self.token_ids.sort_unstable();
+            let mut iter = self.token_ids.iter();
+            let mut current = *iter.next().unwrap();
+            let mut count = 1u32;
+            for &token_id in iter {
+                if token_id == current {
+                    count += 1;
+                    continue;
                 }
+
                 let (old_posting_memory_size, new_posting_memory_size) = {
                     let posting_list = &mut self.builder.posting_lists[current as usize];
                     let old_posting_memory_size = posting_list.size();
@@ -1499,27 +1611,35 @@ impl IndexWorker {
                 };
                 posting_memory_delta +=
                     new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
-            }
-            self.last_token_count = self.token_ids.len();
-            self.trim_temporary_buffers();
-            self.adjust_tracked_memory_size(
-                old_temporary_memory_size,
-                self.temporary_memory_size(),
-            );
 
-            if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes {
-                return Err(Error::invalid_input(format!(
-                    "single document row_id={} exceeds worker memory limit: {} > {} bytes",
-                    row_id, self.memory_size, self.worker_memory_limit_bytes
-                )));
+                current = token_id;
+                count = 1;
             }
+            let (old_posting_memory_size, new_posting_memory_size) = {
+                let posting_list = &mut self.builder.posting_lists[current as usize];
+                let old_posting_memory_size = posting_list.size();
+                posting_list.add(doc_id, PositionRecorder::Count(count));
+                let new_posting_memory_size = posting_list.size();
+                (old_posting_memory_size, new_posting_memory_size)
+            };
+            posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64;
+            Self::apply_delta(&mut self.memory_size, posting_memory_delta);
+        }
+        self.last_token_count = self.token_ids.len();
+        self.trim_temporary_buffers();
+        self.adjust_tracked_memory_size(old_temporary_memory_size, self.temporary_memory_size());
 
-            if self.builder.docs.len() as u32 == u32::MAX
-                || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes)
-            {
-                self.flush().await?;
-            }
+        if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes {
+            return Err(Error::invalid_input(format!(
+                "single document row_id={} exceeds worker memory limit: {} > {} bytes",
+                row_id, self.memory_size, self.worker_memory_limit_bytes
+            )));
+        }
+
+        if self.builder.docs.len() as u32 == u32::MAX
+            || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes)
+        {
+            self.flush().await?;
         }
 
         Ok(())
@@ -1538,15 +1658,17 @@ impl IndexWorker {
         self.memory_size = self.temporary_memory_size();
         let with_position = self.has_position();
         let format_version = self.builder.format_version;
+        let block_size = self.builder.block_size;
         let builder = std::mem::replace(
             &mut self.builder,
-            InnerBuilder::new_with_format_version(
+            InnerBuilder::new_with_format_version_and_block_size(
                 self.id_alloc
                     .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
                     | self.fragment_mask.unwrap_or(0),
                 with_position,
                 self.token_set_format,
                 format_version,
+                block_size,
             ),
         );
         let written_partition_id = builder.id();
@@ -1666,17 +1788,56 @@ pub fn inverted_list_schema_for_version(
     with_position: bool,
     format_version: InvertedListFormatVersion,
 ) -> SchemaRef {
+    inverted_list_schema_for_version_with_block_size(
+        with_position,
+        format_version,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+pub fn inverted_list_schema_for_version_with_block_size(
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+) -> SchemaRef {
+    inverted_list_schema_for_version_with_block_size_and_impacts(
+        with_position,
+        format_version,
+        block_size,
+        true,
+    )
+}
+
+pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts(
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+    with_impacts: bool,
+) -> SchemaRef {
+    validate_format_version_block_size(format_version, block_size)
+        .expect("invalid FTS format version for posting block size");
     match format_version {
-        InvertedListFormatVersion::V1 => inverted_list_schema_v1(with_position),
-        InvertedListFormatVersion::V2 => inverted_list_schema_with_tail_codec_and_position_codec(
-            with_position,
-            PostingTailCodec::VarintDelta,
-            Some(PositionStreamCodec::PackedDelta),
-        ),
+        InvertedListFormatVersion::V1 => {
+            inverted_list_schema_v1(with_position, block_size, with_impacts)
+        }
+        InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => {
+            inverted_list_schema_with_tail_codec_and_position_codec(
+                with_position,
+                format_version,
+                PostingTailCodec::VarintDelta,
+                Some(PositionStreamCodec::PackedDelta),
+                block_size,
+                with_impacts,
+            )
+        }
     }
 }
 
-fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
+fn inverted_list_schema_v1(
+    with_position: bool,
+    block_size: usize,
+    with_impacts: bool,
+) -> SchemaRef {
     let mut fields = vec![
         arrow_schema::Field::new(
             POSTING_COL,
@@ -1690,6 +1851,17 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
         arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false),
         arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false),
     ];
+    if with_impacts {
+        fields.push(arrow_schema::Field::new(
+            IMPACT_COL,
+            datatypes::DataType::List(Arc::new(Field::new(
+                "item",
+                datatypes::DataType::LargeBinary,
+                true,
+            ))),
+            false,
+        ));
+    }
     if with_position {
         fields.push(arrow_schema::Field::new(
             POSITION_COL,
@@ -1705,24 +1877,44 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
             false,
         ));
     }
-    Arc::new(arrow_schema::Schema::new(fields))
+    Arc::new(arrow_schema::Schema::new_with_metadata(
+        fields,
+        HashMap::from([
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                InvertedListFormatVersion::V1.index_version().to_string(),
+            ),
+        ]),
+    ))
 }
 
 pub fn inverted_list_schema_with_tail_codec(
     with_position: bool,
     posting_tail_codec: PostingTailCodec,
 ) -> SchemaRef {
+    let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+        posting_tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+    .expect("invalid posting tail codec for posting block size");
     inverted_list_schema_with_tail_codec_and_position_codec(
         with_position,
+        format_version,
         posting_tail_codec,
         Some(PositionStreamCodec::PackedDelta),
+        LEGACY_BLOCK_SIZE,
+        false,
     )
 }
 
 fn inverted_list_schema_with_tail_codec_and_position_codec(
     with_position: bool,
+    format_version: InvertedListFormatVersion,
     posting_tail_codec: PostingTailCodec,
     position_codec: Option<PositionStreamCodec>,
+    block_size: usize,
+    with_impacts: bool,
 ) -> SchemaRef {
     let mut fields = vec![
         // we compress the posting lists (including row ids and frequencies),
@@ -1739,6 +1931,17 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
         arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false),
         arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false),
     ];
+    if with_impacts {
+        fields.push(arrow_schema::Field::new(
+            IMPACT_COL,
+            datatypes::DataType::List(Arc::new(Field::new(
+                "item",
+                datatypes::DataType::LargeBinary,
+                true,
+            ))),
+            false,
+        ));
+    }
     if with_position {
         fields.push(arrow_schema::Field::new(
             COMPRESSED_POSITION_COL,
@@ -1759,6 +1962,11 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
         POSTING_TAIL_CODEC_KEY.to_owned(),
         posting_tail_codec.as_str().to_owned(),
     )]);
+    metadata.insert(
+        FTS_FORMAT_VERSION_KEY.to_owned(),
+        format_version.index_version().to_string(),
+    );
+    metadata.insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string());
     if let Some(position_codec) = position_codec.filter(|_| with_position) {
         metadata.insert(
             POSITIONS_LAYOUT_KEY.to_owned(),
@@ -1772,129 +1980,6 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
     Arc::new(arrow_schema::Schema::new_with_metadata(fields, metadata))
 }
 
-/// Flatten the string list stream into a string stream
-pub struct FlattenStream {
-    /// Inner record batch stream with 2 columns:
-    /// 1. doc_col: List(Utf8) or List(LargeUtf8)
-    /// 2. row_id_col: UInt64
-    inner: SendableRecordBatchStream,
-    field_type: DataType,
-    data_type: DataType,
-}
-
-impl FlattenStream {
-    pub fn new(input: SendableRecordBatchStream) -> Self {
-        let schema = input.schema();
-        let field = schema.field(0);
-        let data_type = match field.data_type() {
-            DataType::List(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8,
-            DataType::List(f) if matches!(f.data_type(), DataType::LargeUtf8) => {
-                DataType::LargeUtf8
-            }
-            DataType::LargeList(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8,
-            DataType::LargeList(f) if matches!(f.data_type(), DataType::LargeUtf8) => {
-                DataType::LargeUtf8
-            }
-            _ => panic!(
-                "expect data type List(Utf8) or List(LargeUtf8) but got {:?}",
-                field.data_type()
-            ),
-        };
-        Self {
-            inner: input,
-            field_type: field.data_type().clone(),
-            data_type,
-        }
-    }
-}
-
-impl Stream for FlattenStream {
-    type Item = datafusion_common::Result<RecordBatch>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        match Pin::new(&mut self.inner).poll_next(cx) {
-            Poll::Ready(Some(Ok(batch))) => {
-                let doc_col = batch.column(0);
-                let batch = match self.field_type {
-                    DataType::List(_) => flatten_string_list::<i32>(&batch, doc_col).map_err(|e| {
-                        datafusion_common::error::DataFusionError::Execution(format!(
-                            "flatten string list error: {}",
-                            e
-                        ))
-                    }),
-                    DataType::LargeList(_) => {
-                        flatten_string_list::<i64>(&batch, doc_col).map_err(|e| {
-                            datafusion_common::error::DataFusionError::Execution(format!(
-                                "flatten string list error: {}",
-                                e
-                            ))
-                        })
-                    }
-                    _ => unreachable!(
-                        "expect data type List or LargeList but got {:?}",
-                        self.field_type
-                    ),
-                };
-                Poll::Ready(Some(batch))
-            }
-            Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
-            Poll::Ready(None) => Poll::Ready(None),
-            Poll::Pending => Poll::Pending,
-        }
-    }
-}
-
-impl RecordBatchStream for FlattenStream {
-    fn schema(&self) -> SchemaRef {
-        let schema = Schema::new(vec![
-            Field::new(
-                self.inner.schema().field(0).name(),
-                self.data_type.clone(),
-                true,
-            ),
-            ROW_ID_FIELD.clone(),
-        ]);
-
-        Arc::new(schema)
-    }
-}
-
-fn flatten_string_list<Offset: arrow::array::OffsetSizeTrait>(
-    batch: &RecordBatch,
-    doc_col: &Arc<dyn Array>,
-) -> Result<RecordBatch> {
-    let docs = doc_col.as_list::<Offset>();
-    let row_ids = batch[ROW_ID].as_primitive::<datatypes::UInt64Type>();
-
-    let row_ids = row_ids
-        .values()
-        .iter()
-        .zip(docs.iter())
-        .flat_map(|(row_id, doc)| std::iter::repeat_n(*row_id, doc.map(|d| d.len()).unwrap_or(0)));
-
-    let row_ids = Arc::new(UInt64Array::from_iter_values(row_ids));
-    let docs = match docs.value_type() {
-        datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => docs.values().clone(),
-        _ => {
-            return Err(Error::index(format!(
-                "expect data type String or LargeString but got {}",
-                docs.value_type()
-            )));
-        }
-    };
-
-    let schema = Schema::new(vec![
-        Field::new(
-            batch.schema().field(0).name(),
-            docs.data_type().clone(),
-            true,
-        ),
-        ROW_ID_FIELD.clone(),
-    ]);
-    let batch = RecordBatch::try_new(Arc::new(schema), vec![docs, row_ids])?;
-    Ok(batch)
-}
-
 pub(crate) fn token_file_path(partition_id: u64) -> String {
     format!("part_{}_{}", partition_id, TOKENS_FILE)
 }
@@ -2015,7 +2100,6 @@ async fn merge_metadata_files(
     let mut params = None;
     let mut token_set_format = None;
     let mut format_version = None;
-    let mut posting_tail_codec = None;
     let mut deleted_fragments = RoaringBitmap::new();
     progress
         .stage_start(
@@ -2057,9 +2141,6 @@ async fn merge_metadata_files(
         if format_version.is_none() {
             format_version = Some(parse_format_version_from_metadata(metadata)?);
         }
-        if posting_tail_codec.is_none() {
-            posting_tail_codec = Some(parse_posting_tail_codec(metadata)?);
-        }
 
         if reader.num_rows() > 0 {
             let metadata_batch = reader.read_range(0..1, None).await?;
@@ -2125,8 +2206,7 @@ async fn merge_metadata_files(
         None,
         deleted_fragments,
     )
-    .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1))
-    .with_posting_tail_codec(posting_tail_codec.unwrap_or(PostingTailCodec::Fixed32));
+    .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1));
     progress
         .stage_start("write_merged_metadata", Some(1), "files")
         .await?;
@@ -2168,7 +2248,7 @@ pub fn document_input(
         DataType::List(field) | DataType::LargeList(field)
             if matches!(field.data_type(), DataType::Utf8 | DataType::LargeUtf8) =>
         {
-            Ok(Box::pin(FlattenStream::new(input)))
+            Ok(input)
         }
         DataType::LargeBinary => match field.metadata().get(ARROW_EXT_NAME_KEY) {
             Some(name) if name.as_str() == JSON_EXT_NAME => {
@@ -2192,8 +2272,10 @@ pub fn document_input(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::Index;
     use crate::metrics::NoOpMetricsCollector;
     use crate::progress::IndexBuildProgress;
+    use crate::scalar::inverted::{MemBM25Scorer, Scorer};
     use crate::scalar::{IndexFile, IndexReader, IndexWriter, ScalarIndex};
     use arrow_array::{RecordBatch, StringArray, UInt64Array};
     use arrow_schema::{DataType, Field, Schema};
@@ -2226,6 +2308,17 @@ mod tests {
         RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap()
     }
 
+    fn make_doc_batch_from_docs(docs: Vec<Option<&str>>) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("doc", DataType::Utf8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let num_rows = docs.len();
+        let docs = Arc::new(StringArray::from(docs));
+        let row_ids = Arc::new(UInt64Array::from_iter_values(0..num_rows as u64));
+        RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap()
+    }
+
     struct FailingListObjectStore {
         inner: InMemory,
     }
@@ -2613,8 +2706,7 @@ mod tests {
         }
 
         async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
-            // The posting-list writer stores the group offsets as a global
-            // buffer; mirror the real writer's 1-indexed return value.
+            // Mirror the real writer's 1-indexed return value.
             Ok(1)
         }
 
@@ -2699,63 +2791,6 @@ mod tests {
         }
     }
 
-    fn collect_group_starts(config: PostingGroupConfig, sizes: &[usize]) -> Vec<u32> {
-        let mut acc = PostingGroupAccumulator::new(config);
-        for &size in sizes {
-            acc.push(size);
-        }
-        acc.into_starts()
-    }
-
-    #[test]
-    fn test_group_accumulator_cuts_on_target_bytes() {
-        let config = PostingGroupConfig {
-            target_bytes: 100,
-            max_tokens: 1000,
-        };
-        // 40+40 -> cut at 80? no, 80 < 100; third 40 reaches 120 >= 100 -> cut.
-        // So group 0 = tokens [0,3), then a new group starts at token 3.
-        let starts = collect_group_starts(config, &[40, 40, 40, 10, 10]);
-        assert_eq!(starts, vec![0, 3]);
-    }
-
-    #[test]
-    fn test_group_accumulator_cuts_on_max_tokens() {
-        let config = PostingGroupConfig {
-            target_bytes: 1_000_000,
-            max_tokens: 2,
-        };
-        // Byte target never reached; cap of 2 forces a cut every 2 tokens.
-        let starts = collect_group_starts(config, &[1, 1, 1, 1, 1]);
-        assert_eq!(starts, vec![0, 2, 4]);
-    }
-
-    #[test]
-    fn test_group_accumulator_clamps_oversized_term() {
-        let config = PostingGroupConfig {
-            target_bytes: 100,
-            max_tokens: 64,
-        };
-        // A term larger than the target that *starts* a group occupies that
-        // group alone ([1, 2) here), so a single huge posting list is never
-        // forced to share a cache entry. Token 0 (==100) closes its own group
-        // first; the trailing small terms regroup after the big one.
-        let starts = collect_group_starts(config, &[100, 5000, 10, 10]);
-        assert_eq!(starts, vec![0, 1, 2]);
-
-        // A huge term encountered mid-group is absorbed and closes that group;
-        // we never split one term across groups.
-        let starts = collect_group_starts(config, &[10, 10, 5000, 10, 10]);
-        assert_eq!(starts, vec![0, 3]);
-    }
-
-    #[test]
-    fn test_group_accumulator_empty_and_single() {
-        let config = PostingGroupConfig::default();
-        assert_eq!(collect_group_starts(config, &[]), Vec::<u32>::new());
-        assert_eq!(collect_group_starts(config, &[10]), vec![0]);
-    }
-
     #[tokio::test]
     async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> {
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
@@ -2890,6 +2925,10 @@ mod tests {
         expected_partitions.dedup();
         let remapped_partitions = (0..expected_partitions.len() as u64).collect::<Vec<_>>();
         assert_eq!(written_partitions, remapped_partitions);
+        assert_eq!(
+            parse_format_version_from_metadata(metadata)?,
+            InvertedListFormatVersion::V2
+        );
 
         for (new_id, old_id) in expected_partitions.iter().enumerate() {
             assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?;
@@ -3241,6 +3280,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format,
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: params.block_size,
             },
         )
         .await?;
@@ -3264,6 +3304,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format,
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: params.block_size,
             },
         )
         .await?;
@@ -3411,6 +3452,39 @@ mod tests {
         assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta);
     }
 
+    #[test]
+    fn test_v3_128_reuses_v2_physical_layout() {
+        for with_position in [false, true] {
+            for with_impacts in [false, true] {
+                let v2 = inverted_list_schema_for_version_with_block_size_and_impacts(
+                    with_position,
+                    InvertedListFormatVersion::V2,
+                    LEGACY_BLOCK_SIZE,
+                    with_impacts,
+                );
+                let v3 = inverted_list_schema_for_version_with_block_size_and_impacts(
+                    with_position,
+                    InvertedListFormatVersion::V3,
+                    LEGACY_BLOCK_SIZE,
+                    with_impacts,
+                );
+
+                assert_eq!(v2.fields(), v3.fields());
+                let mut v2_metadata = v2.metadata.clone();
+                let mut v3_metadata = v3.metadata.clone();
+                assert_eq!(
+                    v2_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(),
+                    Some("2")
+                );
+                assert_eq!(
+                    v3_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(),
+                    Some("3")
+                );
+                assert_eq!(v2_metadata, v3_metadata);
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> {
         let index_dir = TempDir::default();
@@ -3459,6 +3533,126 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_zero_token_string_documents_are_skipped_in_corpus_stats() -> Result<()> {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch_from_docs(vec![
+            Some(""),
+            Some("   "),
+            Some("the"),
+            Some("overlength"),
+            None,
+            Some("hello"),
+        ]);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let params =
+            InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English)
+                .with_position(false)
+                .remove_stop_words(true)
+                .stem(false)
+                .max_token_length(Some(6))
+                .num_workers(1);
+
+        let mut builder = InvertedIndexBuilder::new(params);
+        builder
+            .update(Box::pin(stream), store.as_ref(), None)
+            .await?;
+
+        let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?;
+        let (total_tokens, num_docs, token_docs) =
+            index.bm25_stats_for_terms(&["hello".to_string()]).await?;
+        assert_eq!(total_tokens, 1);
+        assert_eq!(num_docs, 1);
+        assert_eq!(token_docs, vec![1]);
+
+        let actual_scorer = MemBM25Scorer::new(
+            total_tokens,
+            num_docs,
+            HashMap::from([("hello".to_string(), token_docs[0])]),
+        );
+        let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)]));
+        assert_eq!(
+            actual_scorer.avg_doc_length(),
+            expected_scorer.avg_doc_length()
+        );
+        assert_eq!(
+            actual_scorer.query_weight("hello"),
+            expected_scorer.query_weight("hello")
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_all_empty_string_documents_build_empty_index() -> Result<()> {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let batch = make_doc_batch_from_docs(vec![Some(""), Some("   "), None]);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let params =
+            InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English)
+                .with_position(false)
+                .remove_stop_words(false)
+                .stem(false)
+                .max_token_length(None)
+                .num_workers(1);
+
+        let mut builder = InvertedIndexBuilder::new(params);
+        builder
+            .update(Box::pin(stream), store.as_ref(), None)
+            .await?;
+
+        let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?;
+        assert!(index.partitions.is_empty());
+        let statistics = index.statistics()?;
+        assert_eq!(statistics["num_tokens"], 0);
+        assert_eq!(statistics["num_docs"], 0);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_all_empty_string_documents_do_not_create_tail_partition() -> Result<()> {
+        let tokenizer = InvertedIndexParams::default().build()?;
+        let store = Arc::new(CountingStore::new());
+        let id_alloc = Arc::new(AtomicU64::new(0));
+        let mut worker = IndexWorker::new(
+            tokenizer,
+            store,
+            id_alloc,
+            IndexWorkerConfig {
+                with_position: false,
+                format_version: InvertedListFormatVersion::V1,
+                fragment_mask: None,
+                token_set_format: TokenSetFormat::default(),
+                worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
+            },
+        )
+        .await?;
+
+        worker
+            .process_batch(make_doc_batch_from_docs(vec![Some(""), Some("   "), None]))
+            .await?;
+        let output = worker.finish().await?;
+
+        assert!(output.partitions.is_empty());
+        assert!(output.tail_partition.is_none());
+
+        Ok(())
+    }
+
     lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>);
 
     #[derive(Debug, Default)]
@@ -3746,6 +3940,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
@@ -3777,6 +3972,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
@@ -3815,6 +4011,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
diff --git a/rust/lance-index/src/scalar/inverted/cache_codec.rs b/rust/lance-index/src/scalar/inverted/cache_codec.rs
index a676455d5c9..e59b22dc501 100644
--- a/rust/lance-index/src/scalar/inverted/cache_codec.rs
+++ b/rust/lance-index/src/scalar/inverted/cache_codec.rs
@@ -14,9 +14,13 @@
 //! - the compressed posting list: an IPC section for `blocks`, then the
 //!   position sections (legacy IPC, or shared block-offsets IPC + a raw blob of
 //!   the [`SharedPositionStream`] byte buffer, which has its own portable
-//!   encoding);
+//!   encoding), then an optional impact IPC section;
 //! - the plain posting list: an IPC section of `(row_ids, frequencies)`, then
 //!   an optional legacy position IPC section;
+//! - a packed posting-list group: one IPC section containing the original
+//!   `List<LargeBinary>` posting rows and optional impact rows; prewarmed groups
+//!   omit score/length metadata and inject it from the posting reader into
+//!   query-local views;
 //! - the standalone [`Positions`] codec: the position sections alone.
 //!
 //! All sections read back zero-copy via [`lance_arrow::ipc`]. This is the FTS
@@ -39,10 +43,13 @@ use crate::cache_pb::{
     PostingTailCodec as PbPostingTailCodec,
 };
 
+use super::impact::ImpactSkipData;
 use super::index::{
     CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec,
-    Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream,
+    Positions, PostingList, PostingListGroup, PostingListGroupStorage, PostingTailCodec,
+    SharedPositionStream,
 };
+use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size};
 
 // ---------------------------------------------------------------------------
 // Tags
@@ -50,6 +57,8 @@ use super::index::{
 
 const POSTING_VARIANT_PLAIN: u8 = 0;
 const POSTING_VARIANT_COMPRESSED: u8 = 1;
+const GROUP_VARIANT_MATERIALIZED: u8 = 0;
+const GROUP_VARIANT_PACKED: u8 = 1;
 
 // ---------------------------------------------------------------------------
 // Codec enum mappings
@@ -72,6 +81,23 @@ fn proto_to_posting_tail_codec(c: PbPostingTailCodec) -> PostingTailCodec {
     }
 }
 
+fn posting_tail_codec_to_tag(c: PostingTailCodec) -> u8 {
+    match c {
+        PostingTailCodec::Fixed32 => 0,
+        PostingTailCodec::VarintDelta => 1,
+    }
+}
+
+fn posting_tail_codec_from_tag(tag: u8) -> Result<PostingTailCodec> {
+    match tag {
+        0 => Ok(PostingTailCodec::Fixed32),
+        1 => Ok(PostingTailCodec::VarintDelta),
+        other => Err(Error::io(format!(
+            "unknown packed posting tail codec: {other}"
+        ))),
+    }
+}
+
 fn position_stream_codec_to_proto(c: PositionStreamCodec) -> PbPositionStreamCodec {
     match c {
         PositionStreamCodec::VarintDocDelta => PbPositionStreamCodec::VarintDocDelta,
@@ -95,6 +121,7 @@ const BLOCK_OFFSETS_COLUMN: &str = "block_offsets";
 const ROW_IDS_COLUMN: &str = "row_ids";
 const FREQUENCIES_COLUMN: &str = "frequencies";
 const BLOCKS_COLUMN: &str = "blocks";
+const IMPACTS_COLUMN: &str = "impacts";
 
 fn legacy_positions_batch(list: &ListArray) -> Result<RecordBatch> {
     let schema = Arc::new(Schema::new(vec![Field::new(
@@ -108,7 +135,8 @@ fn legacy_positions_batch(list: &ListArray) -> Result<RecordBatch> {
 fn read_legacy_positions(r: &mut CacheEntryReader<'_>) -> Result<ListArray> {
     let batch = r.read_ipc()?;
     Ok(batch
-        .column(0)
+        .column_by_name(POSITION_LIST_COLUMN)
+        .ok_or_else(|| Error::io("legacy position column is missing".to_string()))?
         .as_any()
         .downcast_ref::<ListArray>()
         .ok_or_else(|| Error::io("legacy position column is not a ListArray".to_string()))?
@@ -156,7 +184,8 @@ fn read_position_sections(
         PbPositionStorage::Shared => {
             let batch = r.read_ipc()?;
             let block_offsets = batch
-                .column(0)
+                .column_by_name(BLOCK_OFFSETS_COLUMN)
+                .ok_or_else(|| Error::io("block_offsets column is missing".to_string()))?
                 .as_primitive_opt::<UInt32Type>()
                 .ok_or_else(|| Error::io("block_offsets column is not UInt32".to_string()))?
                 .values()
@@ -178,7 +207,11 @@ fn read_position_sections(
 
 impl CacheCodecImpl for PostingList {
     const TYPE_ID: &'static str = "lance.fts.PostingList";
-    const CURRENT_VERSION: u32 = 1;
+    // Version 3 adds the optional impact IPC section. Main already used v2 for
+    // configurable posting block sizes, so impact data needs a distinct
+    // version to keep older readers from accepting a body with an extra
+    // section they cannot consume.
+    const CURRENT_VERSION: u32 = 3;
 
     fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         match self {
@@ -194,15 +227,24 @@ impl CacheCodecImpl for PostingList {
     }
 
     fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
-        let variant = r.read_u8()?;
-        match variant {
-            POSTING_VARIANT_PLAIN => Ok(Self::Plain(deserialize_plain(r)?)),
-            POSTING_VARIANT_COMPRESSED => Ok(Self::Compressed(deserialize_compressed(r)?)),
-            other => Err(Error::io(format!("unknown PostingList variant: {other}"))),
+        match r.version() {
+            1 | 2 | Self::CURRENT_VERSION => deserialize_posting_list_body(r),
+            other => Err(Error::io(format!(
+                "unsupported PostingList cache version: {other}"
+            ))),
         }
     }
 }
 
+fn deserialize_posting_list_body(r: &mut CacheEntryReader<'_>) -> Result<PostingList> {
+    let variant = r.read_u8()?;
+    match variant {
+        POSTING_VARIANT_PLAIN => Ok(PostingList::Plain(deserialize_plain(r)?)),
+        POSTING_VARIANT_COMPRESSED => Ok(PostingList::Compressed(deserialize_compressed(r)?)),
+        other => Err(Error::io(format!("unknown PostingList variant: {other}"))),
+    }
+}
+
 fn serialize_plain(w: &mut CacheEntryWriter<'_>, plain: &PlainPostingList) -> Result<()> {
     // Plain postings carry only per-doc legacy positions (or none).
     let position_storage = if plain.positions.is_some() {
@@ -236,13 +278,15 @@ fn deserialize_plain(r: &mut CacheEntryReader<'_>) -> Result<PlainPostingList> {
 
     let batch = r.read_ipc()?;
     let row_ids = batch
-        .column(0)
+        .column_by_name(ROW_IDS_COLUMN)
+        .ok_or_else(|| Error::io("row_ids column is missing".to_string()))?
         .as_primitive_opt::<UInt64Type>()
         .ok_or_else(|| Error::io("row_ids column is not UInt64".to_string()))?
         .values()
         .clone();
     let frequencies = batch
-        .column(1)
+        .column_by_name(FREQUENCIES_COLUMN)
+        .ok_or_else(|| Error::io("frequencies column is missing".to_string()))?
         .as_primitive_opt::<Float32Type>()
         .ok_or_else(|| Error::io("frequencies column is not Float32".to_string()))?
         .values()
@@ -291,6 +335,8 @@ fn serialize_compressed(
         posting_tail_codec: posting_tail_codec_to_proto(posting.posting_tail_codec) as i32,
         position_storage: position_storage as i32,
         position_stream_codec: position_stream_codec as i32,
+        block_size: posting.block_size as u32,
+        has_impacts: posting.impacts.is_some(),
     };
     w.write_header(&header)?;
 
@@ -305,6 +351,15 @@ fn serialize_compressed(
     if let Some(storage) = &posting.positions {
         write_position_sections(w, storage)?;
     }
+    if let Some(impacts) = &posting.impacts {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            IMPACTS_COLUMN,
+            DataType::LargeBinary,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(impacts.entries().clone())])?;
+        w.write_ipc(&batch)?;
+    }
     Ok(())
 }
 
@@ -314,7 +369,8 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result<CompressedPost
 
     let batch = r.read_ipc()?;
     let blocks = batch
-        .column(0)
+        .column_by_name(BLOCKS_COLUMN)
+        .ok_or_else(|| Error::io("blocks column is missing".to_string()))?
         .as_any()
         .downcast_ref::<LargeBinaryArray>()
         .ok_or_else(|| Error::io("blocks column is not a LargeBinaryArray".to_string()))?
@@ -322,13 +378,33 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result<CompressedPost
 
     let stream_codec = proto_to_position_stream_codec(header.position_stream_codec());
     let positions = read_position_sections(r, header.position_storage(), stream_codec)?;
+    let block_size = if header.block_size == 0 {
+        LEGACY_BLOCK_SIZE
+    } else {
+        validate_block_size(header.block_size as usize)?
+    };
+    let impacts = if r.version() >= 3 && header.has_impacts {
+        let batch = r.read_ipc()?;
+        let entries = batch
+            .column_by_name(IMPACTS_COLUMN)
+            .ok_or_else(|| Error::io("impacts column is missing".to_string()))?
+            .as_any()
+            .downcast_ref::<LargeBinaryArray>()
+            .ok_or_else(|| Error::io("impacts column is not a LargeBinaryArray".to_string()))?
+            .clone();
+        Some(ImpactSkipData::new(entries, blocks.len())?)
+    } else {
+        None
+    };
 
     Ok(CompressedPostingList::new(
         blocks,
         header.max_score,
         header.length,
         posting_tail_codec,
+        block_size,
         positions,
+        impacts,
     ))
 }
 
@@ -336,34 +412,79 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result<CompressedPost
 // PostingListGroup codec
 // ---------------------------------------------------------------------------
 
-/// Serializes a [`PostingListGroup`] as a member-count header followed by each
-/// member posting list written **inline** through the same writer. Reusing the
-/// [`PostingList`] codec inline (rather than into per-member sub-buffers) keeps
-/// each member's Arrow IPC sections 64-byte aligned within the group entry, so
-/// they read back zero-copy. Member bodies are self-delimiting, so they need no
-/// length prefixes to separate them. See issue #7040.
+/// Version 2 distinguishes packed groups from the materialized fallback. A
+/// packed group writes one IPC batch; materialized groups retain the v1 inline
+/// member framing used by legacy and position-bearing prewarm paths. Version 3
+/// marks packed groups whose IPC schema can carry configurable posting block
+/// sizes while retaining the version-2 body framing. Version 4 adds optional
+/// impact data, either as an inline section for materialized members or as a
+/// column in the packed IPC batch.
 impl CacheCodecImpl for PostingListGroup {
     const TYPE_ID: &'static str = "lance.fts.PostingListGroup";
-    const CURRENT_VERSION: u32 = 1;
+    const CURRENT_VERSION: u32 = 4;
 
     fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
-        let count = u32::try_from(self.posting_lists.len())
+        let count = u32::try_from(self.len())
             .map_err(|_| Error::io("posting list group too large to serialize".to_string()))?;
-        w.write_header(&PostingListGroupHeader { count })?;
-        for posting in &self.posting_lists {
-            posting.serialize(w)?;
+        match &self.storage {
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                w.write_u8(GROUP_VARIANT_MATERIALIZED)?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                for posting in posting_lists {
+                    posting.serialize(w)?;
+                }
+            }
+            PostingListGroupStorage::Packed(group) => {
+                w.write_u8(GROUP_VARIANT_PACKED)?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                w.write_u8(posting_tail_codec_to_tag(group.posting_tail_codec))?;
+                w.write_ipc(&group.batch)?;
+            }
         }
         Ok(())
     }
 
     fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
-        let header: PostingListGroupHeader = r.read_header()?;
-        let mut posting_lists = Vec::with_capacity(header.count as usize);
-        for _ in 0..header.count {
-            posting_lists.push(PostingList::deserialize(r)?);
+        match r.version() {
+            1 => return deserialize_materialized_group(r),
+            2 | 3 | Self::CURRENT_VERSION => {}
+            other => {
+                return Err(Error::io(format!(
+                    "unsupported PostingListGroup cache version: {other}"
+                )));
+            }
         }
-        Ok(Self::new(posting_lists))
+
+        let variant = r.read_u8()?;
+        match variant {
+            GROUP_VARIANT_MATERIALIZED => deserialize_materialized_group(r),
+            GROUP_VARIANT_PACKED => {
+                let header: PostingListGroupHeader = r.read_header()?;
+                let posting_tail_codec = posting_tail_codec_from_tag(r.read_u8()?)?;
+                let batch = r.read_ipc()?;
+                if batch.num_rows() != header.count as usize {
+                    return Err(Error::io(format!(
+                        "packed posting group row count {} does not match header count {}",
+                        batch.num_rows(),
+                        header.count
+                    )));
+                }
+                Self::new_packed(batch, posting_tail_codec)
+            }
+            other => Err(Error::io(format!(
+                "unknown PostingListGroup variant: {other}"
+            ))),
+        }
+    }
+}
+
+fn deserialize_materialized_group(r: &mut CacheEntryReader<'_>) -> Result<PostingListGroup> {
+    let header: PostingListGroupHeader = r.read_header()?;
+    let mut posting_lists = Vec::with_capacity(header.count as usize);
+    for _ in 0..header.count {
+        posting_lists.push(deserialize_posting_list_body(r)?);
     }
+    Ok(PostingListGroup::new(posting_lists))
 }
 
 // ---------------------------------------------------------------------------
@@ -409,17 +530,26 @@ impl CacheCodecImpl for Positions {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
     use arrow::buffer::ScalarBuffer;
-    use arrow_array::LargeBinaryArray;
-    use arrow_array::builder::{Int32Builder, ListBuilder};
+    use arrow_array::builder::{Int32Builder, LargeBinaryBuilder, ListBuilder};
+    use arrow_array::{Array, LargeBinaryArray, RecordBatch};
+    use arrow_schema::{Field, Schema};
     use bytes::Bytes;
     use lance_core::Result;
     use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
 
+    use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec};
+
+    use super::super::impact::{ImpactSkipData, ImpactSkipDataBuilder};
     use super::super::index::{
-        CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec,
-        Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream,
+        CompressedPositionStorage, CompressedPostingList, IMPACT_COL, POSTING_BLOCK_SIZE_KEY,
+        POSTING_COL, PlainPostingList, PositionStreamCodec, Positions, PostingList,
+        PostingListGroup, PostingTailCodec, SharedPositionStream,
     };
+    use super::super::tokenizer::LEGACY_BLOCK_SIZE;
 
     fn legacy_positions(rows: &[&[i32]]) -> arrow_array::ListArray {
         let mut builder = ListBuilder::new(Int32Builder::new());
@@ -432,6 +562,73 @@ mod tests {
         builder.finish()
     }
 
+    fn packed_batch(postings: &[Vec<Vec<u8>>], block_size: Option<usize>) -> RecordBatch {
+        let mut builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for posting in postings {
+            for block in posting {
+                builder.values().append_value(block);
+            }
+            builder.append(true);
+        }
+        let postings = builder.finish();
+        let fields = vec![Field::new(POSTING_COL, postings.data_type().clone(), false)];
+        let schema = Arc::new(match block_size {
+            Some(block_size) => Schema::new_with_metadata(
+                fields,
+                HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string())]),
+            ),
+            None => Schema::new(fields),
+        });
+        RecordBatch::try_new(schema, vec![Arc::new(postings)]).unwrap()
+    }
+
+    fn packed_group(
+        postings: &[Vec<Vec<u8>>],
+        posting_tail_codec: PostingTailCodec,
+        block_size: Option<usize>,
+    ) -> PostingListGroup {
+        PostingListGroup::new_packed(packed_batch(postings, block_size), posting_tail_codec)
+            .unwrap()
+    }
+
+    fn packed_group_with_impacts(
+        postings: &[Vec<Vec<u8>>],
+        impacts: &[ImpactSkipData],
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> PostingListGroup {
+        assert_eq!(postings.len(), impacts.len());
+        let posting_batch = packed_batch(postings, Some(block_size));
+        let mut impacts_builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for impacts in impacts {
+            for entry_idx in 0..impacts.entries().len() {
+                impacts_builder
+                    .values()
+                    .append_value(impacts.entries().value(entry_idx));
+            }
+            impacts_builder.append(true);
+        }
+        let impacts = impacts_builder.finish();
+        let fields = vec![
+            Field::new(
+                POSTING_COL,
+                posting_batch.column(0).data_type().clone(),
+                false,
+            ),
+            Field::new(IMPACT_COL, impacts.data_type().clone(), false),
+        ];
+        let schema = Arc::new(Schema::new_with_metadata(
+            fields,
+            posting_batch.schema_ref().metadata().clone(),
+        ));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![posting_batch.column(0).clone(), Arc::new(impacts)],
+        )
+        .unwrap();
+        PostingListGroup::new_packed(batch, posting_tail_codec).unwrap()
+    }
+
     fn assert_plain_eq(a: &PlainPostingList, b: &PlainPostingList) {
         assert_eq!(a.row_ids.as_ref(), b.row_ids.as_ref());
         assert_eq!(a.frequencies.as_ref(), b.frequencies.as_ref());
@@ -461,6 +658,20 @@ mod tests {
         }
     }
 
+    fn impact_skip_data(level0_len: usize, block_size: usize) -> ImpactSkipData {
+        let mut builder = ImpactSkipDataBuilder::with_capacity(level0_len, block_size);
+        for block_idx in 0..level0_len {
+            let doc_base = block_idx as u32 * 10;
+            builder
+                .append_block(&[
+                    (doc_base + 1, block_idx as u32 + 1, 10),
+                    (doc_base + 9, block_idx as u32 + 2, 8),
+                ])
+                .unwrap();
+        }
+        builder.finish().unwrap()
+    }
+
     /// Serialize a codec body (no envelope) into a standalone buffer.
     fn body_bytes<T: CacheCodecImpl>(entry: &T) -> Bytes {
         let mut buf = Vec::new();
@@ -475,6 +686,34 @@ mod tests {
         T::deserialize(&mut r)
     }
 
+    fn from_body_version<T: CacheCodecImpl>(data: &Bytes, version: u32) -> Result<T> {
+        let mut r = CacheEntryReader::new(data, 0, version);
+        T::deserialize(&mut r)
+    }
+
+    fn compressed_body_with_ipc_sections(
+        blocks: &RecordBatch,
+        impacts: Option<&RecordBatch>,
+    ) -> Bytes {
+        let mut buf = Vec::new();
+        let mut w = CacheEntryWriter::new(&mut buf);
+        w.write_u8(super::POSTING_VARIANT_COMPRESSED).unwrap();
+        w.write_header(&CompressedPostingHeader {
+            max_score: 1.0,
+            length: 1,
+            posting_tail_codec: PbPostingTailCodec::VarintDelta as i32,
+            block_size: 256,
+            has_impacts: impacts.is_some(),
+            ..Default::default()
+        })
+        .unwrap();
+        w.write_ipc(blocks).unwrap();
+        if let Some(impacts) = impacts {
+            w.write_ipc(impacts).unwrap();
+        }
+        Bytes::from(buf)
+    }
+
     fn roundtrip_posting_list(entry: &PostingList) -> PostingList {
         from_body::<PostingList>(&body_bytes(entry)).unwrap()
     }
@@ -532,14 +771,22 @@ mod tests {
             Some(&[1u8, 2, 3, 4, 5][..]),
             Some(&[6, 7, 8, 9, 10][..]),
         ]);
-        let posting =
-            CompressedPostingList::new(blocks, 3.5, 42, PostingTailCodec::VarintDelta, None);
+        let posting = CompressedPostingList::new(
+            blocks,
+            3.5,
+            42,
+            PostingTailCodec::VarintDelta,
+            256,
+            None,
+            None,
+        );
         let entry = PostingList::Compressed(posting.clone());
         match roundtrip_posting_list(&entry) {
             PostingList::Compressed(restored) => {
                 assert_eq!(restored.max_score, posting.max_score);
                 assert_eq!(restored.length, posting.length);
                 assert_eq!(restored.posting_tail_codec, posting.posting_tail_codec);
+                assert_eq!(restored.block_size, posting.block_size);
                 assert_eq!(restored.blocks, posting.blocks);
                 assert!(restored.positions.is_none());
             }
@@ -547,6 +794,73 @@ mod tests {
         }
     }
 
+    #[test]
+    fn compressed_posting_list_impacts_roundtrip() {
+        let blocks = LargeBinaryArray::from_opt_vec(vec![
+            Some(&[1u8, 2, 3, 4, 5][..]),
+            Some(&[6, 7, 8, 9, 10][..]),
+        ]);
+        let impacts = impact_skip_data(blocks.len(), 256);
+        let posting = CompressedPostingList::new(
+            blocks,
+            3.5,
+            42,
+            PostingTailCodec::VarintDelta,
+            256,
+            None,
+            Some(impacts.clone()),
+        );
+        let entry = PostingList::Compressed(posting);
+        match roundtrip_posting_list(&entry) {
+            PostingList::Compressed(restored) => {
+                let restored = restored.impacts.expect("impacts should roundtrip");
+                assert_eq!(restored.level0_len(), impacts.level0_len());
+                assert_eq!(restored.level1_len(), impacts.level1_len());
+                assert_eq!(restored.entries(), impacts.entries());
+            }
+            PostingList::Plain(_) => panic!("expected Compressed variant"),
+        }
+    }
+
+    #[test]
+    fn compressed_posting_list_missing_ipc_columns_returns_error() {
+        let empty = RecordBatch::new_empty(Arc::new(Schema::empty()));
+        assert!(
+            from_body::<PostingList>(&compressed_body_with_ipc_sections(&empty, None)).is_err()
+        );
+
+        let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1_u8, 2, 3][..])]);
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            super::BLOCKS_COLUMN,
+            blocks.data_type().clone(),
+            false,
+        )]));
+        let blocks = RecordBatch::try_new(schema, vec![Arc::new(blocks)]).unwrap();
+        assert!(
+            from_body::<PostingList>(&compressed_body_with_ipc_sections(&blocks, Some(&empty)))
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn compressed_posting_list_v1_cache_without_impacts_decodes() {
+        let posting = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]),
+            1.25,
+            5,
+            PostingTailCodec::Fixed32,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+            None,
+            None,
+        );
+        let data = body_bytes(&PostingList::Compressed(posting));
+        let restored = from_body_version::<PostingList>(&data, 1).unwrap();
+        let PostingList::Compressed(restored) = restored else {
+            panic!("expected Compressed variant");
+        };
+        assert!(restored.impacts.is_none());
+    }
+
     #[test]
     fn compressed_posting_list_legacy_positions_roundtrip() {
         let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]);
@@ -555,9 +869,11 @@ mod tests {
             1.25,
             5,
             PostingTailCodec::Fixed32,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
             Some(CompressedPositionStorage::LegacyPerDoc(legacy_positions(
                 &[&[0, 4, 8]],
             ))),
+            None,
         );
         let entry = PostingList::Compressed(posting.clone());
         match roundtrip_posting_list(&entry) {
@@ -589,7 +905,9 @@ mod tests {
                 7.0,
                 3,
                 PostingTailCodec::VarintDelta,
+                256,
                 Some(CompressedPositionStorage::SharedStream(stream)),
+                None,
             );
             let entry = PostingList::Compressed(posting.clone());
             match roundtrip_posting_list(&entry) {
@@ -617,9 +935,11 @@ mod tests {
             7.0,
             3,
             PostingTailCodec::VarintDelta,
+            256,
             Some(CompressedPositionStorage::SharedStream(
                 expected_stream.clone(),
             )),
+            None,
         );
         let serialized = body_bytes(&PostingList::Compressed(posting));
 
@@ -651,6 +971,8 @@ mod tests {
             2.5,
             7,
             PostingTailCodec::VarintDelta,
+            256,
+            None,
             None,
         ));
 
@@ -661,9 +983,11 @@ mod tests {
         ] {
             let group = PostingListGroup::new(members.clone());
             let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
-            assert_eq!(restored.posting_lists.len(), members.len());
-            for (a, b) in members.iter().zip(restored.posting_lists.iter()) {
-                match (a, b) {
+            assert!(!restored.is_packed());
+            assert_eq!(restored.len(), members.len());
+            for (index, a) in members.iter().enumerate() {
+                let b = restored.posting_list(index, None, None).unwrap().unwrap();
+                match (a, &b) {
                     (PostingList::Plain(x), PostingList::Plain(y)) => assert_plain_eq(x, y),
                     (PostingList::Compressed(x), PostingList::Compressed(y)) => {
                         assert_eq!(x.blocks, y.blocks);
@@ -676,6 +1000,167 @@ mod tests {
         }
     }
 
+    #[test]
+    fn packed_posting_list_group_roundtrip_and_v1_fallback() {
+        let group = packed_group(
+            &[vec![vec![1, 2, 3], vec![4, 5]], vec![vec![7; 16 * 1024]]],
+            PostingTailCodec::VarintDelta,
+            Some(256),
+        );
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(restored.is_packed());
+        assert_eq!(restored.len(), 2);
+        for slot in 0..2 {
+            let max_score = [1.5, 3.25][slot];
+            let length = [3, 4096][slot];
+            let expected = group
+                .posting_list(slot, Some(max_score), Some(length))
+                .unwrap()
+                .unwrap();
+            let actual = restored
+                .posting_list(slot, Some(max_score), Some(length))
+                .unwrap()
+                .unwrap();
+            let (PostingList::Compressed(expected), PostingList::Compressed(actual)) =
+                (expected, actual)
+            else {
+                panic!("expected compressed packed posting views");
+            };
+            assert_eq!(actual.blocks, expected.blocks);
+            assert_eq!(actual.max_score, expected.max_score);
+            assert_eq!(actual.length, expected.length);
+            assert_eq!(actual.posting_tail_codec, expected.posting_tail_codec);
+            assert_eq!(actual.block_size, 256);
+        }
+
+        let legacy_packed =
+            packed_group(&[vec![vec![9, 8, 7]]], PostingTailCodec::VarintDelta, None);
+        let restored = from_body::<PostingListGroup>(&body_bytes(&legacy_packed)).unwrap();
+        let PostingList::Compressed(posting) = restored
+            .posting_list(0, Some(2.0), Some(3))
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed legacy packed posting");
+        };
+        assert_eq!(posting.block_size, LEGACY_BLOCK_SIZE);
+
+        let legacy_member = PostingList::Compressed(CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]),
+            2.0,
+            3,
+            PostingTailCodec::VarintDelta,
+            LEGACY_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let mut legacy_body = Vec::new();
+        let mut writer = CacheEntryWriter::new(&mut legacy_body);
+        writer
+            .write_header(&crate::cache_pb::PostingListGroupHeader { count: 1 })
+            .unwrap();
+        legacy_member.serialize(&mut writer).unwrap();
+        let legacy_body = Bytes::from(legacy_body);
+        let mut reader = CacheEntryReader::new(&legacy_body, 0, 1);
+        let restored = PostingListGroup::deserialize(&mut reader).unwrap();
+        assert!(!restored.is_packed());
+        assert_eq!(restored.len(), 1);
+    }
+
+    #[test]
+    fn posting_list_group_impacted_compressed_members_roundtrip() {
+        let first = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..]), Some(&[4u8, 5, 6][..])]),
+            3.0,
+            256,
+            PostingTailCodec::VarintDelta,
+            LEGACY_BLOCK_SIZE,
+            None,
+            Some(impact_skip_data(2, LEGACY_BLOCK_SIZE)),
+        );
+        let second = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[7u8, 8, 9][..])]),
+            5.0,
+            128,
+            PostingTailCodec::Fixed32,
+            256,
+            Some(CompressedPositionStorage::SharedStream(
+                SharedPositionStream::new(
+                    PositionStreamCodec::PackedDelta,
+                    vec![0u32, 12],
+                    Bytes::from(vec![0xABu8; 32]),
+                ),
+            )),
+            Some(impact_skip_data(1, 256)),
+        );
+        let members = vec![
+            PostingList::Compressed(first.clone()),
+            PostingList::Compressed(second.clone()),
+        ];
+        let group = PostingListGroup::new(members);
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(!restored.is_packed());
+        assert_eq!(restored.len(), 2);
+
+        let expected = [&first, &second];
+        for (slot, expected) in expected.iter().enumerate() {
+            let restored = restored.posting_list(slot, None, None).unwrap().unwrap();
+            let PostingList::Compressed(restored) = restored else {
+                panic!("expected compressed member");
+            };
+            assert_eq!(restored.blocks, expected.blocks);
+            assert_eq!(restored.length, expected.length);
+            assert_eq!(restored.max_score, expected.max_score);
+            assert_eq!(restored.posting_tail_codec, expected.posting_tail_codec);
+            assert_eq!(restored.block_size, expected.block_size);
+            assert_eq!(
+                restored.impacts.as_ref().unwrap().entries(),
+                expected.impacts.as_ref().unwrap().entries()
+            );
+            match (&expected.positions, &restored.positions) {
+                (Some(expected), Some(restored)) => {
+                    assert_position_storage_eq(expected, restored);
+                }
+                (None, None) => {}
+                _ => panic!("position storage mismatch"),
+            }
+        }
+    }
+
+    #[test]
+    fn packed_posting_list_group_impacts_roundtrip() {
+        let postings = vec![vec![vec![1, 2, 3], vec![4, 5, 6]], vec![vec![7, 8, 9]]];
+        let expected_impacts = vec![impact_skip_data(2, 256), impact_skip_data(1, 256)];
+        let group = packed_group_with_impacts(
+            &postings,
+            &expected_impacts,
+            PostingTailCodec::VarintDelta,
+            256,
+        );
+
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(restored.is_packed());
+        assert_eq!(restored.len(), expected_impacts.len());
+        for (slot, expected) in expected_impacts.iter().enumerate() {
+            let posting = restored
+                .posting_list(slot, Some(3.0), Some(256))
+                .unwrap()
+                .unwrap();
+            let PostingList::Compressed(posting) = posting else {
+                panic!("expected compressed packed posting");
+            };
+            let actual = posting.impacts.as_ref().expect("impacts should roundtrip");
+            assert_eq!(actual.entries(), expected.entries());
+            assert_eq!(actual.level0_len(), expected.level0_len());
+            assert_eq!(
+                actual.level1_doc_up_to(0),
+                expected.level1_doc_up_to(0),
+                "impact entries should remain decodable with the packed block size",
+            );
+            assert!(actual.level1_doc_up_to(0).is_some());
+        }
+    }
+
     #[test]
     fn positions_legacy_roundtrip() {
         let positions = Positions(CompressedPositionStorage::LegacyPerDoc(legacy_positions(
@@ -717,26 +1202,130 @@ mod tests {
         use std::sync::Arc;
 
         use arrow_array::Array;
-        use lance_core::cache::CacheCodec;
+        use arrow_schema::DataType;
+        use lance_core::cache::{
+            CacheCodec, CacheCodecImpl, CacheDecode, CacheEntryReader, CacheEntryWriter,
+            CacheMissReason,
+        };
+        use lance_core::{Error, Result};
         use prost::Message;
 
+        use super::super::{
+            BLOCKS_COLUMN, GROUP_VARIANT_PACKED, POSTING_VARIANT_COMPRESSED,
+            posting_tail_codec_to_tag,
+        };
         use super::*;
-        use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec};
+        use crate::cache_pb::{
+            CompressedPostingHeader, PostingListGroupHeader, PostingTailCodec as PbPostingTailCodec,
+        };
 
         type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
 
+        struct PostingListV2Codec(PostingList);
+
+        impl CacheCodecImpl for PostingListV2Codec {
+            const TYPE_ID: &'static str = <PostingList as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 2;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                self.0.serialize(w)
+            }
+
+            fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                PostingList::deserialize(r).map(Self)
+            }
+        }
+
+        struct PostingListGroupV3Codec(PostingListGroup);
+
+        impl CacheCodecImpl for PostingListGroupV3Codec {
+            const TYPE_ID: &'static str = <PostingListGroup as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 3;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                self.0.serialize(w)
+            }
+
+            fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                PostingListGroup::deserialize(r).map(Self)
+            }
+        }
+
+        struct LegacyCompressedPostingV1 {
+            blocks: LargeBinaryArray,
+        }
+
+        impl CacheCodecImpl for LegacyCompressedPostingV1 {
+            const TYPE_ID: &'static str = <PostingList as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 1;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                w.write_u8(POSTING_VARIANT_COMPRESSED)?;
+                w.write_header(&CompressedPostingHeader {
+                    max_score: 2.0,
+                    length: 3,
+                    posting_tail_codec: PbPostingTailCodec::VarintDelta as i32,
+                    ..Default::default()
+                })?;
+                let schema = Arc::new(Schema::new(vec![Field::new(
+                    BLOCKS_COLUMN,
+                    DataType::LargeBinary,
+                    false,
+                )]));
+                let batch = RecordBatch::try_new(schema, vec![Arc::new(self.blocks.clone())])?;
+                w.write_ipc(&batch)
+            }
+
+            fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                Err(Error::io(
+                    "LegacyCompressedPostingV1 is a writer-only test codec".to_string(),
+                ))
+            }
+        }
+
+        struct LegacyPackedGroupV2 {
+            batch: RecordBatch,
+            posting_tail_codec: PostingTailCodec,
+        }
+
+        impl CacheCodecImpl for LegacyPackedGroupV2 {
+            const TYPE_ID: &'static str = <PostingListGroup as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 2;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                w.write_u8(GROUP_VARIANT_PACKED)?;
+                let count = u32::try_from(self.batch.num_rows())
+                    .map_err(|_| Error::io("legacy packed group is too large".to_string()))?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                w.write_u8(posting_tail_codec_to_tag(self.posting_tail_codec))?;
+                w.write_ipc(&self.batch)
+            }
+
+            fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                Err(Error::io(
+                    "LegacyPackedGroupV2 is a writer-only test codec".to_string(),
+                ))
+            }
+        }
+
         fn codec() -> CacheCodec {
             CacheCodec::from_impl::<PostingList>()
         }
 
-        /// Serialize an entry through the full codec (envelope + body).
-        fn serialize_entry(entry: PostingList) -> Vec<u8> {
+        fn serialize_typed_entry<T: CacheCodecImpl + 'static>(entry: T) -> Vec<u8> {
             let any: ArcAny = Arc::new(entry);
             let mut buf = Vec::new();
-            codec().serialize(&any, &mut buf).unwrap();
+            CacheCodec::from_impl::<T>()
+                .serialize(&any, &mut buf)
+                .unwrap();
             buf
         }
 
+        /// Serialize an entry through the full codec (envelope + body).
+        fn serialize_entry(entry: PostingList) -> Vec<u8> {
+            serialize_typed_entry(entry)
+        }
+
         /// A `Bytes` whose base address is 64-byte aligned, modelling a backend
         /// that reads cache entries into an aligned buffer.
         fn aligned_bytes(payload: &[u8]) -> Bytes {
@@ -760,7 +1349,9 @@ mod tests {
                 7.0,
                 3,
                 PostingTailCodec::VarintDelta,
+                256,
                 Some(CompressedPositionStorage::SharedStream(stream)),
+                None,
             ))
         }
 
@@ -795,25 +1386,17 @@ mod tests {
             assert!(points_in(stream.bytes().as_ptr() as usize));
         }
 
-        /// Every member of a `PostingListGroup` must also decode zero-copy. The
-        /// group writes its members inline so each member's IPC sections stay
-        /// 64-byte aligned within the entry; embedding members in per-member
-        /// sub-buffers would land them at arbitrary offsets and force a
-        /// realigning memcpy on load.
+        /// A packed group's single IPC batch and all posting views decoded from
+        /// it must borrow the cache entry's aligned input buffer.
         #[test]
-        fn group_member_sections_are_zero_copy_through_envelope() {
-            let make_member = |fill: u8| {
-                let blocks =
-                    LargeBinaryArray::from_opt_vec(vec![Some(&[fill; 48][..]), Some(&[fill; 48])]);
-                PostingList::Compressed(CompressedPostingList::new(
-                    blocks,
-                    7.0,
-                    3,
-                    PostingTailCodec::VarintDelta,
-                    None,
-                ))
-            };
-            let group = PostingListGroup::new(vec![make_member(9), make_member(1)]);
+        fn packed_group_sections_are_zero_copy_through_envelope() {
+            let postings = vec![
+                vec![vec![9; 48], vec![9; 48]],
+                vec![vec![1; 48], vec![1; 48]],
+            ];
+            let impacts = vec![impact_skip_data(2, 256), impact_skip_data(2, 256)];
+            let group =
+                packed_group_with_impacts(&postings, &impacts, PostingTailCodec::VarintDelta, 256);
 
             let group_codec = CacheCodec::from_impl::<PostingListGroup>();
             let any: ArcAny = Arc::new(group);
@@ -828,8 +1411,13 @@ mod tests {
             let end = base + serialized.len();
             let points_in = |ptr: usize| ptr >= base && ptr < end;
 
-            assert_eq!(restored.posting_lists.len(), 2);
-            for member in &restored.posting_lists {
+            assert!(restored.is_packed());
+            assert_eq!(restored.len(), 2);
+            for slot in 0..restored.len() {
+                let member = restored
+                    .posting_list(slot, Some(7.0), Some(3))
+                    .unwrap()
+                    .unwrap();
                 let PostingList::Compressed(member) = member else {
                     panic!("expected Compressed member");
                 };
@@ -837,7 +1425,17 @@ mod tests {
                     assert!(
                         points_in(buf.as_ptr() as usize),
                         "group member blocks buffer was realigned out of the input — \
-                         misaligned IPC section",
+                        misaligned IPC section",
+                    );
+                }
+                let impacts = member
+                    .impacts
+                    .as_ref()
+                    .expect("packed impacts should decode");
+                for buf in impacts.entries().to_data().buffers() {
+                    assert!(
+                        points_in(buf.as_ptr() as usize),
+                        "group member impact buffer was realigned out of the input",
                     );
                 }
             }
@@ -915,6 +1513,110 @@ mod tests {
             assert!(codec().deserialize(&Bytes::from(buf)).hit().is_none());
         }
 
+        #[test]
+        fn old_codecs_reject_new_impact_envelopes_as_version_too_new() {
+            let posting = Bytes::from(serialize_entry(compressed_with_shared_positions()));
+            match CacheCodec::from_impl::<PostingListV2Codec>().deserialize(&posting) {
+                CacheDecode::Miss(reason) => {
+                    assert_eq!(reason, CacheMissReason::VersionTooNew)
+                }
+                CacheDecode::Hit(_) => panic!("v2 PostingList codec accepted a v3 envelope"),
+            }
+
+            let group = packed_group(
+                &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]],
+                PostingTailCodec::VarintDelta,
+                Some(256),
+            );
+            let group = Bytes::from(serialize_typed_entry(group));
+            match CacheCodec::from_impl::<PostingListGroupV3Codec>().deserialize(&group) {
+                CacheDecode::Miss(reason) => {
+                    assert_eq!(reason, CacheMissReason::VersionTooNew)
+                }
+                CacheDecode::Hit(_) => {
+                    panic!("v3 PostingListGroup codec accepted a v4 envelope")
+                }
+            }
+        }
+
+        #[test]
+        fn current_codecs_read_previous_main_versions() {
+            let previous_posting = PostingListV2Codec(compressed_with_shared_positions());
+            let previous_posting = Bytes::from(serialize_typed_entry(previous_posting));
+            let restored = codec().deserialize(&previous_posting).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Compressed(restored) = restored.as_ref() else {
+                panic!("expected compressed posting");
+            };
+            assert_eq!(restored.block_size, 256);
+            assert!(restored.impacts.is_none());
+
+            let previous_group = PostingListGroupV3Codec(packed_group(
+                &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]],
+                PostingTailCodec::VarintDelta,
+                Some(256),
+            ));
+            let previous_group = Bytes::from(serialize_typed_entry(previous_group));
+            let restored = CacheCodec::from_impl::<PostingListGroup>()
+                .deserialize(&previous_group)
+                .hit()
+                .unwrap()
+                .downcast::<PostingListGroup>()
+                .unwrap();
+            assert!(restored.is_packed());
+            assert_eq!(restored.len(), 2);
+            let PostingList::Compressed(restored) = restored
+                .posting_list(0, Some(2.0), Some(3))
+                .unwrap()
+                .unwrap()
+            else {
+                panic!("expected compressed packed posting");
+            };
+            assert_eq!(restored.block_size, 256);
+            assert!(restored.impacts.is_none());
+        }
+
+        #[test]
+        fn current_codecs_read_legacy_payloads_without_block_size() {
+            let legacy_posting = LegacyCompressedPostingV1 {
+                blocks: LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]),
+            };
+            let legacy_posting = Bytes::from(serialize_typed_entry(legacy_posting));
+            let restored = codec().deserialize(&legacy_posting).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Compressed(restored) = restored.as_ref() else {
+                panic!("expected a compressed legacy posting");
+            };
+            assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE);
+
+            let legacy_batch = packed_batch(&[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], None);
+            assert!(
+                !legacy_batch
+                    .schema_ref()
+                    .metadata()
+                    .contains_key(POSTING_BLOCK_SIZE_KEY)
+            );
+            let legacy_group = LegacyPackedGroupV2 {
+                batch: legacy_batch,
+                posting_tail_codec: PostingTailCodec::VarintDelta,
+            };
+            let legacy_group = Bytes::from(serialize_typed_entry(legacy_group));
+            let restored = CacheCodec::from_impl::<PostingListGroup>()
+                .deserialize(&legacy_group)
+                .hit()
+                .unwrap()
+                .downcast::<PostingListGroup>()
+                .unwrap();
+            let PostingList::Compressed(restored) = restored
+                .posting_list(0, Some(2.0), Some(3))
+                .unwrap()
+                .unwrap()
+            else {
+                panic!("expected a compressed legacy packed posting");
+            };
+            assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE);
+        }
+
         /// A pre-stabilization blob (no magic) self-heals to a miss.
         #[test]
         fn pre_stabilization_blob_is_miss() {
diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs
index d75fa742ee7..6d57b032650 100644
--- a/rust/lance-index/src/scalar/inverted/encoding.rs
+++ b/rust/lance-index/src/scalar/inverted/encoding.rs
@@ -5,10 +5,15 @@ use std::io::Write;
 
 use super::builder::BLOCK_SIZE;
 use super::index::{PositionStreamCodec, PostingTailCodec};
+#[cfg(test)]
+use super::tokenizer::LEGACY_BLOCK_SIZE;
+use super::tokenizer::validate_block_size;
 use arrow::array::LargeBinaryBuilder;
-use lance_bitpacking::{BitPacker, BitPacker4x};
+use lance_bitpacking::{BitPacker, BitPacker4x, BitPacker8x};
 use lance_core::{Error, Result};
 
+pub const MAX_POSTING_BLOCK_SIZE: usize = BitPacker8x::BLOCK_LEN;
+
 // we compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE),
 // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies)
 // each block is:
@@ -41,18 +46,40 @@ pub fn compress_posting_list<'a>(
 
 #[cfg(test)]
 pub fn compress_posting_list_with_tail_codec<'a>(
+    length: usize,
+    doc_ids: impl Iterator<Item = &'a u32>,
+    frequencies: impl Iterator<Item = &'a u32>,
+    block_max_scores: impl Iterator<Item = f32>,
+    tail_codec: PostingTailCodec,
+) -> Result<arrow::array::LargeBinaryArray> {
+    compress_posting_list_with_tail_codec_and_block_size(
+        length,
+        doc_ids,
+        frequencies,
+        block_max_scores,
+        tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+#[cfg(test)]
+pub fn compress_posting_list_with_tail_codec_and_block_size<'a>(
     length: usize,
     doc_ids: impl Iterator<Item = &'a u32>,
     frequencies: impl Iterator<Item = &'a u32>,
     mut block_max_scores: impl Iterator<Item = f32>,
     tail_codec: PostingTailCodec,
+    block_size: usize,
 ) -> Result<arrow::array::LargeBinaryArray> {
-    if length < BLOCK_SIZE {
+    let block_size = validate_block_size(block_size)?;
+    if length < block_size {
         // directly do remainder compression to avoid overhead of creating buffer
         let mut builder = LargeBinaryBuilder::with_capacity(1, length * 4 * 2 + 1);
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
         compress_posting_remainder(
             doc_ids.copied().collect::<Vec<_>>().as_slice(),
             frequencies.copied().collect::<Vec<_>>().as_slice(),
@@ -63,27 +90,25 @@ pub fn compress_posting_list_with_tail_codec<'a>(
         return Ok(builder.finish());
     }
 
-    let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(BLOCK_SIZE), length * 3);
-    let mut buffer = [0u8; BLOCK_SIZE * 4 + 5];
-    let mut doc_id_buffer = Vec::with_capacity(BLOCK_SIZE);
-    let mut freq_buffer = Vec::with_capacity(BLOCK_SIZE);
+    let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(block_size), length * 3);
+    let mut doc_id_buffer = Vec::with_capacity(block_size);
+    let mut freq_buffer = Vec::with_capacity(block_size);
     for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) {
         doc_id_buffer.push(*doc_id);
         freq_buffer.push(*freq);
 
-        if doc_id_buffer.len() < BLOCK_SIZE {
+        if doc_id_buffer.len() < block_size {
             continue;
         }
 
-        assert_eq!(doc_id_buffer.len(), BLOCK_SIZE);
+        assert_eq!(doc_id_buffer.len(), block_size);
 
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
-        // delta encoding + bitpacking for doc ids
-        compress_sorted_block(&doc_id_buffer, &mut buffer, &mut builder)?;
-        // bitpacking for frequencies
-        compress_block(&freq_buffer, &mut buffer, &mut builder)?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
+        encode_posting_block_payload(&doc_id_buffer, &freq_buffer, &mut builder)?;
         builder.append_value("");
         doc_id_buffer.clear();
         freq_buffer.clear();
@@ -91,44 +116,187 @@ pub fn compress_posting_list_with_tail_codec<'a>(
 
     // we don't compress the last block if it is not full
     if !doc_id_buffer.is_empty() {
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
         compress_posting_remainder(&doc_id_buffer, &freq_buffer, tail_codec, &mut builder)?;
         builder.append_value("");
     }
     Ok(builder.finish())
 }
 
+/// Byte length of the block-max-score prefix on posting blocks. 128-doc
+/// blocks store a per-block max score, patched in at build time; 256-document
+/// blocks always carry impact skip data, which supersedes it, so they
+/// store none.
+#[inline]
+pub fn posting_block_score_prefix_len(block_size: usize) -> usize {
+    if block_size == MAX_POSTING_BLOCK_SIZE {
+        0
+    } else {
+        4
+    }
+}
+
 pub fn encode_full_posting_block_into(
     doc_ids: &[u32],
     frequencies: &[u32],
     block: &mut Vec<u8>,
 ) -> Result<()> {
-    debug_assert_eq!(doc_ids.len(), BLOCK_SIZE);
-    debug_assert_eq!(frequencies.len(), BLOCK_SIZE);
-    block.extend_from_slice(&0f32.to_le_bytes());
-    let mut buffer = [0u8; BLOCK_SIZE * 4 + 5];
-    compress_sorted_block(doc_ids, &mut buffer, block)?;
-    compress_block(frequencies, &mut buffer, block)?;
+    validate_block_size(doc_ids.len())?;
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    if posting_block_score_prefix_len(doc_ids.len()) > 0 {
+        block.extend_from_slice(&0f32.to_le_bytes());
+    }
+    encode_posting_block_payload(doc_ids, frequencies, block)?;
     Ok(())
 }
 
+fn encode_posting_block_payload(
+    doc_ids: &[u32],
+    frequencies: &[u32],
+    block: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    validate_block_size(doc_ids.len())?;
+    let mut buffer = [0u8; MAX_POSTING_BLOCK_SIZE * 4 + 5];
+    match doc_ids.len() {
+        BitPacker4x::BLOCK_LEN => {
+            compress_sorted_block_with::<BitPacker4x>(doc_ids, &mut buffer, block)?;
+            compress_block_with::<BitPacker4x>(frequencies, &mut buffer, block)?;
+        }
+        // 256-document blocks store frequencies with patched FOR:
+        // outliers no longer widen the whole block, which matters because one
+        // large tf per block otherwise doubles the frequency payload.
+        BitPacker8x::BLOCK_LEN => {
+            compress_sorted_block_with::<BitPacker8x>(doc_ids, &mut buffer, block)?;
+            compress_pfor_block_with::<BitPacker8x>(frequencies, &mut buffer, block)?;
+        }
+        _ => unreachable!("validated posting block size should be supported"),
+    }
+    Ok(())
+}
+
+/// Patched FOR (Lucene PForUtil style): pick the body bit width that
+/// minimizes total bytes, pack all values masked to that width, and append
+/// up to [`PFOR_MAX_EXCEPTIONS`] exceptions as (index u8, high-bits varint).
+const PFOR_MAX_EXCEPTIONS: usize = 31;
+
+#[inline]
+fn u32_bits(value: u32) -> usize {
+    (32 - value.leading_zeros()) as usize
+}
+
+#[inline]
+fn varint_u32_len(value: u32) -> usize {
+    u32_bits(value).max(1).div_ceil(7)
+}
+
+fn compress_pfor_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let max_bits = data.iter().map(|&v| u32_bits(v)).max().unwrap_or(0);
+    let mut best_width = max_bits;
+    let mut best_cost = P::BLOCK_LEN * max_bits / 8;
+    for width in (0..max_bits).rev() {
+        let mut exceptions = 0usize;
+        let mut exception_bytes = 0usize;
+        for &value in data {
+            if u32_bits(value) > width {
+                exceptions += 1;
+                exception_bytes += 1 + varint_u32_len(value >> width);
+            }
+        }
+        if exceptions > PFOR_MAX_EXCEPTIONS {
+            break;
+        }
+        let cost = P::BLOCK_LEN * width / 8 + exception_bytes;
+        if cost < best_cost {
+            best_cost = cost;
+            best_width = width;
+        }
+    }
+
+    let mask = if best_width >= 32 {
+        u32::MAX
+    } else {
+        (1u32 << best_width) - 1
+    };
+    let mut body = [0u32; MAX_POSTING_BLOCK_SIZE];
+    let mut exception_buf = Vec::new();
+    let mut exception_count = 0u8;
+    for (index, &value) in data.iter().enumerate() {
+        body[index] = value & mask;
+        if u32_bits(value) > best_width {
+            exception_buf.push(index as u8);
+            encode_varint_u32(&mut exception_buf, value >> best_width);
+            exception_count += 1;
+        }
+    }
+    let compressor = P::new();
+    let num_bytes = compressor.compress(&body[..P::BLOCK_LEN], buffer, best_width as u8);
+    let _ = builder.write(&[best_width as u8, exception_count])?;
+    let _ = builder.write(&buffer[..num_bytes])?;
+    let _ = builder.write(&exception_buf)?;
+    Ok(())
+}
+
+fn decompress_pfor_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let width = block[0];
+    let exception_count = block[1] as usize;
+    let compressor = P::new();
+    let num_bytes = compressor.decompress(&block[2..], buffer, width);
+    let mut offset = 2 + num_bytes;
+    for _ in 0..exception_count {
+        let index = block[offset] as usize;
+        offset += 1;
+        let high = decode_varint_u32(block, &mut offset)
+            .expect("pfor exception high bits should be a valid varint");
+        buffer[index] |= high << width;
+    }
+    res.extend_from_slice(buffer);
+    offset
+}
+
 pub fn encode_remainder_posting_block_into(
     doc_ids: &[u32],
     frequencies: &[u32],
     codec: PostingTailCodec,
+    block_size: usize,
     block: &mut Vec<u8>,
 ) -> Result<()> {
     debug_assert_eq!(doc_ids.len(), frequencies.len());
-    block.extend_from_slice(&0f32.to_le_bytes());
+    if posting_block_score_prefix_len(block_size) > 0 {
+        block.extend_from_slice(&0f32.to_le_bytes());
+    }
     compress_posting_remainder(doc_ids, frequencies, codec, block)?;
     Ok(())
 }
 
 #[inline]
 fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
-    let compressor = BitPacker4x::new();
+    compress_sorted_block_with::<BitPacker4x>(data, buffer, builder)
+}
+
+#[inline]
+fn compress_sorted_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let compressor = P::new();
     let num_bits = compressor.num_bits_sorted(data[0], data);
     let num_bytes = compressor.compress_sorted(data[0], data, buffer, num_bits);
     let _ = builder.write(data[0].to_le_bytes().as_ref())?;
@@ -139,7 +307,17 @@ fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Wri
 
 #[inline]
 fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
-    let compressor = BitPacker4x::new();
+    compress_block_with::<BitPacker4x>(data, buffer, builder)
+}
+
+#[inline]
+fn compress_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let compressor = P::new();
     let num_bits = compressor.num_bits(data);
     let num_bytes = compressor.compress(data, buffer, num_bits);
     let _ = builder.write(&[num_bits])?;
@@ -236,7 +414,7 @@ pub fn compress_positions(positions: &[u32]) -> Result<arrow::array::LargeBinary
 }
 
 #[inline]
-fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
+pub fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
     while value >= 0x80 {
         dst.push((value as u8) | 0x80);
         value >>= 7;
@@ -244,39 +422,6 @@ fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
     dst.push(value as u8);
 }
 
-/// Encode a monotonically increasing sequence of `group_starts` (the first
-/// row of each posting-list cache group) as varint-encoded deltas. The first
-/// value is stored as-is and each subsequent value as its delta from the
-/// previous one; since deltas are group sizes (1..=cap) they fit in ~1 byte,
-/// keeping the buffer tiny even for indexes with millions of tokens. See
-/// issue #7040.
-pub(super) fn encode_group_starts(group_starts: &[u32]) -> Vec<u8> {
-    let mut dst = Vec::with_capacity(group_starts.len());
-    let mut previous = 0u32;
-    for &start in group_starts {
-        debug_assert!(start >= previous, "group_starts must be monotonic");
-        encode_varint_u32(&mut dst, start - previous);
-        previous = start;
-    }
-    dst
-}
-
-/// Decode the buffer produced by [`encode_group_starts`] back into the
-/// absolute `group_starts` values.
-pub(super) fn decode_group_starts(src: &[u8]) -> Result<Vec<u32>> {
-    let mut group_starts = Vec::new();
-    let mut offset = 0;
-    let mut previous = 0u32;
-    while offset < src.len() {
-        let delta = decode_varint_u32(src, &mut offset)?;
-        previous = previous
-            .checked_add(delta)
-            .ok_or_else(|| Error::index("group_starts delta decode overflowed u32".to_owned()))?;
-        group_starts.push(previous);
-    }
-    Ok(group_starts)
-}
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub(super) struct PositionBlockBuilder {
     codec: PositionStreamCodec,
@@ -379,7 +524,7 @@ impl PositionBlockBuilder {
 }
 
 #[inline]
-fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result<u32> {
+pub fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result<u32> {
     let mut value = 0u32;
     let mut shift = 0u32;
     while *offset < src.len() {
@@ -557,15 +702,10 @@ fn decode_position_stream_packed_block(
     let mut deltas = Vec::with_capacity(total_positions);
 
     for _ in 0..full_delta_blocks {
-        if packed_offset >= src.len() {
-            return Err(Error::index(
-                "unexpected EOF while decoding packed position stream".to_owned(),
-            ));
-        }
-        let num_bits = src[packed_offset];
-        packed_offset += 1;
-        let consumed = compressor.decompress(&src[packed_offset..], &mut packed_values, num_bits);
-        packed_offset += consumed;
+        let (num_bits, payload, next_offset) = packed_position_group(src, packed_offset)?;
+        let consumed = compressor.decompress(payload, &mut packed_values, num_bits);
+        debug_assert_eq!(consumed, payload.len());
+        packed_offset = next_offset;
         deltas.extend_from_slice(&packed_values);
     }
 
@@ -601,6 +741,144 @@ fn decode_position_stream_packed_block(
     Ok(())
 }
 
+fn packed_position_group(src: &[u8], offset: usize) -> Result<(u8, &[u8], usize)> {
+    let num_bits = *src.get(offset).ok_or_else(|| {
+        Error::index(format!(
+            "unexpected EOF reading packed position group header at byte offset {offset}; \
+             stream length is {}",
+            src.len()
+        ))
+    })?;
+    if num_bits > u32::BITS as u8 {
+        return Err(Error::index(format!(
+            "invalid packed position group bit width {num_bits} at byte offset {offset}; \
+             expected at most {}",
+            u32::BITS
+        )));
+    }
+
+    let payload_start = offset
+        .checked_add(1)
+        .ok_or_else(|| Error::index("packed position group offset overflow".to_owned()))?;
+    let payload_len = usize::from(num_bits) * BLOCK_SIZE / 8;
+    let payload_end = payload_start
+        .checked_add(payload_len)
+        .ok_or_else(|| Error::index("packed position group length overflow".to_owned()))?;
+    let payload = src.get(payload_start..payload_end).ok_or_else(|| {
+        Error::index(format!(
+            "unexpected EOF reading packed position group payload at byte offset {offset}; \
+             need {payload_len} bytes after the header but stream length is {}",
+            src.len()
+        ))
+    })?;
+    Ok((num_bits, payload, payload_end))
+}
+
+/// Decode one document's positions out of a PackedDelta position block
+/// without decoding the rest of the block. Full 128-delta groups are
+/// self-describing (`[num_bits u8][16 * num_bits packed bytes]`), so group
+/// byte offsets are recovered by hopping headers — no format change is
+/// involved. `delta_range` is the doc's range in the block-wide delta stream
+/// (from the frequency prefix sums); per-doc deltas reset at document
+/// boundaries, so decoding starts cleanly at `delta_range.start`.
+///
+/// The caller passes per-block scratch state that this function maintains:
+/// `group_offsets` (lazily extended header index, seeded with `[0]`),
+/// `unpacked_group`/`unpacked_group_idx` (the last unpacked group), and
+/// `tail_cache` (the varint tail, decoded in full on first touch). All are
+/// reset by the caller when the block cursor moves.
+#[allow(clippy::too_many_arguments)]
+pub(super) fn seek_packed_doc_positions(
+    src: &[u8],
+    total_deltas: usize,
+    delta_range: std::ops::Range<usize>,
+    group_offsets: &mut Vec<usize>,
+    unpacked_group: &mut [u32; BLOCK_SIZE],
+    unpacked_group_idx: &mut Option<usize>,
+    tail_cache: &mut Vec<u32>,
+    dst: &mut Vec<u32>,
+) -> Result<()> {
+    dst.clear();
+    if delta_range.start > delta_range.end || delta_range.end > total_deltas {
+        return Err(Error::index(format!(
+            "invalid packed position delta range {}..{} for {total_deltas} total deltas",
+            delta_range.start, delta_range.end
+        )));
+    }
+    if delta_range.is_empty() {
+        return Ok(());
+    }
+    let num_full_groups = total_deltas / BLOCK_SIZE;
+    let packed_deltas_end = num_full_groups * BLOCK_SIZE;
+
+    // Extend the header index far enough for this range (tail needs the
+    // offset one past the last full group).
+    let last_needed_group = if delta_range.end > packed_deltas_end {
+        num_full_groups
+    } else {
+        (delta_range.end - 1) / BLOCK_SIZE
+    };
+    while group_offsets.len() <= last_needed_group {
+        let last = *group_offsets
+            .last()
+            .ok_or_else(|| Error::index("packed position group offsets are empty".to_owned()))?;
+        let (_, _, next_offset) = packed_position_group(src, last)?;
+        group_offsets.push(next_offset);
+    }
+
+    let mut previous = 0u32;
+    let mut first = true;
+    let mut push_delta = |delta: u32, dst: &mut Vec<u32>| -> Result<()> {
+        let position = if first {
+            first = false;
+            delta
+        } else {
+            previous
+                .checked_add(delta)
+                .ok_or_else(|| Error::index("position stream overflow while decoding".to_owned()))?
+        };
+        dst.push(position);
+        previous = position;
+        Ok(())
+    };
+
+    for index in delta_range.start..delta_range.end.min(packed_deltas_end) {
+        let group = index / BLOCK_SIZE;
+        if *unpacked_group_idx != Some(group) {
+            let offset = *group_offsets.get(group).ok_or_else(|| {
+                Error::index(format!(
+                    "missing packed position group offset for group {group}; have {} offsets",
+                    group_offsets.len()
+                ))
+            })?;
+            let (num_bits, payload, _) = packed_position_group(src, offset)?;
+            BitPacker4x::new().decompress(payload, unpacked_group, num_bits);
+            *unpacked_group_idx = Some(group);
+        }
+        push_delta(unpacked_group[index % BLOCK_SIZE], dst)?;
+    }
+
+    if delta_range.end > packed_deltas_end {
+        let tail_len = total_deltas - packed_deltas_end;
+        if tail_cache.len() != tail_len {
+            tail_cache.clear();
+            tail_cache.reserve(tail_len);
+            let mut offset = *group_offsets.get(num_full_groups).ok_or_else(|| {
+                Error::index(format!(
+                    "missing packed position tail offset after {num_full_groups} full groups"
+                ))
+            })?;
+            for _ in 0..tail_len {
+                tail_cache.push(decode_varint_u32(src, &mut offset)?);
+            }
+        }
+        for index in delta_range.start.max(packed_deltas_end)..delta_range.end {
+            push_delta(tail_cache[index - packed_deltas_end], dst)?;
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 pub fn encode_position_stream_block_into(
     positions: &[u32],
@@ -650,23 +928,46 @@ pub fn decompress_posting_list_with_tail_codec(
     posting_list: &arrow::array::LargeBinaryArray,
     tail_codec: PostingTailCodec,
 ) -> Result<(Vec<u32>, Vec<u32>)> {
+    decompress_posting_list_with_tail_codec_and_block_size(
+        num_docs,
+        posting_list,
+        tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+#[cfg(test)]
+pub fn decompress_posting_list_with_tail_codec_and_block_size(
+    num_docs: u32,
+    posting_list: &arrow::array::LargeBinaryArray,
+    tail_codec: PostingTailCodec,
+    block_size: usize,
+) -> Result<(Vec<u32>, Vec<u32>)> {
+    let block_size = validate_block_size(block_size)?;
     let mut doc_ids: Vec<u32> = Vec::with_capacity(num_docs as usize);
     let mut frequencies: Vec<u32> = Vec::with_capacity(num_docs as usize);
 
-    let mut buffer = [0u32; BLOCK_SIZE];
-    let bitpacking_blocks = num_docs as usize / BLOCK_SIZE;
+    let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE];
+    let bitpacking_blocks = num_docs as usize / block_size;
     for compressed in posting_list.iter().take(bitpacking_blocks) {
         let compressed = compressed.unwrap();
-        decompress_posting_block(compressed, &mut buffer, &mut doc_ids, &mut frequencies);
+        decompress_posting_block(
+            compressed,
+            &mut buffer,
+            &mut doc_ids,
+            &mut frequencies,
+            block_size,
+        );
     }
 
-    let remainder = num_docs as usize % BLOCK_SIZE;
+    let remainder = num_docs as usize % block_size;
     if remainder > 0 {
         let compressed = posting_list.value(bitpacking_blocks);
         decompress_posting_remainder(
             compressed,
             remainder,
             tail_codec,
+            block_size,
             &mut doc_ids,
             &mut frequencies,
         );
@@ -701,24 +1002,46 @@ pub fn read_num_positions(compressed: &arrow::array::LargeBinaryArray) -> u32 {
 
 pub fn decompress_posting_block(
     block: &[u8],
-    buffer: &mut [u32; BLOCK_SIZE],
+    buffer: &mut [u32],
     doc_ids: &mut Vec<u32>,
     frequencies: &mut Vec<u32>,
+    block_size: usize,
 ) {
-    // skip the first 4 bytes for the max block score
-    let block = &block[4..];
-    let num_bytes = decompress_sorted_block(block, buffer, doc_ids);
-    decompress_block(&block[num_bytes..], buffer, frequencies);
+    debug_assert!(validate_block_size(block_size).is_ok());
+    debug_assert!(buffer.len() >= block_size);
+    // skip the block max score prefix (128-doc blocks only)
+    let mut block = &block[posting_block_score_prefix_len(block_size)..];
+    match block_size {
+        BitPacker4x::BLOCK_LEN => {
+            let num_bytes = decompress_sorted_block_with::<BitPacker4x>(block, buffer, doc_ids);
+            block = &block[num_bytes..];
+            let num_bytes = decompress_block_with::<BitPacker4x>(block, buffer, frequencies);
+            block = &block[num_bytes..];
+        }
+        BitPacker8x::BLOCK_LEN => {
+            let num_bytes = decompress_sorted_block_with::<BitPacker8x>(block, buffer, doc_ids);
+            block = &block[num_bytes..];
+            let num_bytes = decompress_pfor_block_with::<BitPacker8x>(block, buffer, frequencies);
+            block = &block[num_bytes..];
+        }
+        _ => unreachable!("validated posting block size should be supported"),
+    }
+    debug_assert!(
+        block.is_empty(),
+        "posting block has {} trailing bytes after decoding",
+        block.len()
+    );
 }
 
 pub fn decompress_posting_remainder(
     block: &[u8],
     n: usize,
     codec: PostingTailCodec,
+    block_size: usize,
     doc_ids: &mut Vec<u32>,
     frequencies: &mut Vec<u32>,
 ) {
-    let block = &block[4..];
+    let block = &block[posting_block_score_prefix_len(block_size)..];
     match codec {
         PostingTailCodec::Fixed32 => {
             decompress_raw_remainder(block, n, doc_ids);
@@ -755,9 +1078,14 @@ pub fn decompress_posting_remainder(
     }
 }
 
-pub fn decode_full_posting_block(block: &[u8], doc_ids: &mut Vec<u32>, frequencies: &mut Vec<u32>) {
-    let mut buffer = [0u32; BLOCK_SIZE];
-    decompress_posting_block(block, &mut buffer, doc_ids, frequencies);
+pub fn decode_full_posting_block(
+    block: &[u8],
+    doc_ids: &mut Vec<u32>,
+    frequencies: &mut Vec<u32>,
+    block_size: usize,
+) {
+    let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE];
+    decompress_posting_block(block, &mut buffer, doc_ids, frequencies, block_size);
 }
 
 pub fn decompress_sorted_block(
@@ -765,7 +1093,17 @@ pub fn decompress_sorted_block(
     buffer: &mut [u32; BLOCK_SIZE],
     res: &mut Vec<u32>,
 ) -> usize {
-    let compressor = BitPacker4x::new();
+    decompress_sorted_block_with::<BitPacker4x>(block, buffer, res)
+}
+
+fn decompress_sorted_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let compressor = P::new();
     let initial = u32::from_le_bytes(block[0..4].try_into().unwrap());
     let num_bits = block[4];
     let num_bytes = compressor.decompress_sorted(initial, &block[5..], buffer, num_bits);
@@ -773,11 +1111,18 @@ pub fn decompress_sorted_block(
     5 + num_bytes
 }
 
-fn decompress_block(block: &[u8], buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec<u32>) {
-    let compressor = BitPacker4x::new();
+fn decompress_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let compressor = P::new();
     let num_bits = block[0];
-    compressor.decompress(&block[1..], buffer, num_bits);
+    let num_bytes = compressor.decompress(&block[1..], buffer, num_bits);
     res.extend_from_slice(&buffer[..]);
+    1 + num_bytes
 }
 
 pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>) {
@@ -787,11 +1132,18 @@ pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>
     }
 }
 
-pub fn read_posting_tail_first_doc(block: &[u8], codec: PostingTailCodec) -> u32 {
+pub fn read_posting_tail_first_doc(
+    block: &[u8],
+    codec: PostingTailCodec,
+    block_size: usize,
+) -> u32 {
+    let prefix = posting_block_score_prefix_len(block_size);
     match codec {
-        PostingTailCodec::Fixed32 => u32::from_le_bytes(block[4..8].try_into().unwrap()),
+        PostingTailCodec::Fixed32 => {
+            u32::from_le_bytes(block[prefix..prefix + 4].try_into().unwrap())
+        }
         PostingTailCodec::VarintDelta => {
-            let mut offset = 4usize;
+            let mut offset = prefix;
             decode_varint_u32(block, &mut offset)
                 .expect("posting tail block should contain a valid first doc id")
         }
@@ -805,31 +1157,6 @@ mod tests {
     use itertools::Itertools;
     use rand::Rng;
 
-    #[test]
-    fn test_group_starts_codec_roundtrip() {
-        for case in [
-            vec![],
-            vec![0u32],
-            vec![0, 1, 2, 3],
-            // realistic: monotonic with varied group sizes and a large jump
-            vec![0, 64, 128, 129, 4096, 1_000_000],
-        ] {
-            let encoded = encode_group_starts(&case);
-            let decoded = decode_group_starts(&encoded).unwrap();
-            assert_eq!(decoded, case, "roundtrip mismatch for {case:?}");
-        }
-    }
-
-    #[test]
-    fn test_decode_group_starts_rejects_overflow() {
-        // A crafted buffer whose deltas sum past u32::MAX must error rather
-        // than wrap. Encodes u32::MAX followed by a +1 delta.
-        let mut buf = Vec::new();
-        encode_varint_u32(&mut buf, u32::MAX);
-        encode_varint_u32(&mut buf, 1);
-        assert!(decode_group_starts(&buf).is_err());
-    }
-
     #[test]
     fn test_compress_posting_list() -> Result<()> {
         let num_rows: usize = BLOCK_SIZE * 1024 - 7;
@@ -867,6 +1194,84 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_compress_posting_list_supported_block_sizes() -> Result<()> {
+        for block_size in [128, 256] {
+            let num_rows: usize = block_size * 2 + 7;
+            let doc_ids = (0..num_rows as u32).collect::<Vec<_>>();
+            let frequencies = (0..num_rows as u32)
+                .map(|value| value % 7 + 1)
+                .collect::<Vec<_>>();
+            let block_max_scores =
+                (0..num_rows.div_ceil(block_size)).map(|value| value as f32 + 1.0);
+
+            let posting_list = compress_posting_list_with_tail_codec_and_block_size(
+                doc_ids.len(),
+                doc_ids.iter(),
+                frequencies.iter(),
+                block_max_scores,
+                PostingTailCodec::VarintDelta,
+                block_size,
+            )?;
+            assert_eq!(posting_list.len(), num_rows.div_ceil(block_size));
+
+            let (decoded_doc_ids, decoded_frequencies) =
+                decompress_posting_list_with_tail_codec_and_block_size(
+                    num_rows as u32,
+                    &posting_list,
+                    PostingTailCodec::VarintDelta,
+                    block_size,
+                )?;
+            assert_eq!(decoded_doc_ids, doc_ids);
+            assert_eq!(decoded_frequencies, frequencies);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_256_posting_block_uses_single_physical_bitpack_chunk() -> Result<()> {
+        let block_size = BitPacker8x::BLOCK_LEN;
+        let doc_ids = (0..block_size as u32).collect::<Vec<_>>();
+        let frequencies = (0..block_size as u32)
+            .map(|value| value % 13 + 1)
+            .collect::<Vec<_>>();
+
+        let posting_list = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(1.0),
+            PostingTailCodec::VarintDelta,
+            block_size,
+        )?;
+        assert_eq!(posting_list.len(), 1);
+
+        let block = posting_list.value(0);
+        // 256-doc blocks carry no block-max-score prefix (impacts supply the
+        // per-block bound): [first_doc u32][doc num_bits u8][doc payload]...
+        let doc_num_bits = block[4];
+        let doc_bytes = BitPacker8x::compressed_block_size(doc_num_bits);
+        let freq_header_offset = 5 + doc_bytes;
+        // 256-doc blocks use patched FOR for frequencies:
+        // [width u8][exception_count u8][body][exceptions...]
+        let freq_num_bits = block[freq_header_offset];
+        let exception_count = block[freq_header_offset + 1] as usize;
+        assert_eq!(exception_count, 0, "uniform freqs need no exceptions");
+        let freq_bytes = BitPacker8x::compressed_block_size(freq_num_bits);
+        assert_eq!(block.len(), freq_header_offset + 2 + freq_bytes);
+
+        let (decoded_doc_ids, decoded_frequencies) =
+            decompress_posting_list_with_tail_codec_and_block_size(
+                doc_ids.len() as u32,
+                &posting_list,
+                PostingTailCodec::VarintDelta,
+                block_size,
+            )?;
+        assert_eq!(decoded_doc_ids, doc_ids);
+        assert_eq!(decoded_frequencies, frequencies);
+        Ok(())
+    }
+
     #[test]
     fn test_compress_posting_list_fixed32_tail_still_roundtrips() -> Result<()> {
         let doc_ids = vec![3_u32, 10_u32, 24_u32];
@@ -913,6 +1318,116 @@ mod tests {
         Ok(())
     }
 
+    /// Per-doc seek decoding of a PackedDelta position block must return
+    /// exactly the same positions as decoding the whole block, for every doc,
+    /// across group-boundary-straddling docs and varint tails.
+    #[test]
+    fn test_packed_position_doc_seek_matches_block_decode() -> Result<()> {
+        let mut rng = rand::rng();
+        // Frequency shapes: tiny blocks (tail only), exactly one group, a doc
+        // straddling group boundaries, and a large multi-group block.
+        let freq_shapes: Vec<Vec<u32>> = vec![
+            vec![1],
+            vec![3, 1, 5],
+            vec![64, 64],
+            vec![100, 60, 40],
+            (0..256u32).map(|i| (i % 7) + 1).collect(),
+            vec![300, 2, 129, 1, 77],
+        ];
+        for frequencies in freq_shapes {
+            let total: usize = frequencies.iter().map(|&f| f as usize).sum();
+            // Positions ascend within each doc; docs are independent.
+            let mut positions = Vec::with_capacity(total);
+            for &freq in &frequencies {
+                let mut current = rng.random_range(0..1000u32);
+                for _ in 0..freq {
+                    positions.push(current);
+                    current += rng.random_range(1..50u32);
+                }
+            }
+
+            let mut encoded = Vec::new();
+            encode_position_stream_block_into(
+                &positions,
+                &frequencies,
+                PositionStreamCodec::PackedDelta,
+                &mut encoded,
+            )?;
+
+            let mut whole = Vec::new();
+            decode_position_stream_block(
+                &encoded,
+                &frequencies,
+                PositionStreamCodec::PackedDelta,
+                &mut whole,
+            )?;
+            assert_eq!(whole, positions);
+
+            let mut group_offsets = vec![0usize];
+            let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]);
+            let mut unpacked_group_idx = None;
+            let mut tail_cache = Vec::new();
+            let mut scratch = Vec::new();
+            let mut delta_start = 0usize;
+            for &freq in &frequencies {
+                let delta_end = delta_start + freq as usize;
+                seek_packed_doc_positions(
+                    &encoded,
+                    total,
+                    delta_start..delta_end,
+                    &mut group_offsets,
+                    &mut unpacked_group,
+                    &mut unpacked_group_idx,
+                    &mut tail_cache,
+                    &mut scratch,
+                )?;
+                assert_eq!(
+                    scratch,
+                    whole[delta_start..delta_end],
+                    "doc positions mismatch for range {delta_start}..{delta_end} freqs={frequencies:?}"
+                );
+                delta_start = delta_end;
+            }
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_packed_position_decoders_reject_malformed_groups() {
+        let frequencies = [BLOCK_SIZE as u32];
+        for encoded in [&[1_u8][..], &[33_u8][..]] {
+            let mut decoded = Vec::new();
+            assert!(
+                decode_position_stream_block(
+                    encoded,
+                    &frequencies,
+                    PositionStreamCodec::PackedDelta,
+                    &mut decoded,
+                )
+                .is_err()
+            );
+
+            let mut group_offsets = vec![0usize];
+            let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]);
+            let mut unpacked_group_idx = None;
+            let mut tail_cache = Vec::new();
+            let mut scratch = Vec::new();
+            assert!(
+                seek_packed_doc_positions(
+                    encoded,
+                    BLOCK_SIZE,
+                    0..BLOCK_SIZE,
+                    &mut group_offsets,
+                    &mut unpacked_group,
+                    &mut unpacked_group_idx,
+                    &mut tail_cache,
+                    &mut scratch,
+                )
+                .is_err()
+            );
+        }
+    }
+
     #[test]
     fn test_encode_position_stream_block_roundtrip() -> Result<()> {
         let frequencies = vec![1, 3, 2, 4];
diff --git a/rust/lance-index/src/scalar/inverted/impact.rs b/rust/lance-index/src/scalar/inverted/impact.rs
new file mode 100644
index 00000000000..848c2376459
--- /dev/null
+++ b/rust/lance-index/src/scalar/inverted/impact.rs
@@ -0,0 +1,1055 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::mem::size_of;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use arrow_array::builder::LargeBinaryBuilder;
+use arrow_array::{Array, LargeBinaryArray};
+use lance_core::{Error, Result};
+
+use super::scorer::Scorer;
+
+pub const IMPACT_LEVEL1_BLOCKS: usize = 32;
+const SMALL_FRONTIER_FREQ_LIMIT: usize = 256;
+
+/// On-disk encoding of one impact entry, shared by every posting block size.
+///
+/// Entries contain `[doc_up_to varint][pair_count varint][pairs...]`. Each
+/// pair stores a varint whose high bits are `freq_delta - 1` and whose low bit
+/// reports whether a one-byte norm delta follows. The norm itself is the
+/// quantized `u8` document-length code; the common `norm_delta == 1` case needs
+/// no norm byte.
+#[derive(Debug, Clone)]
+pub struct ImpactSkipData {
+    entries: LargeBinaryArray,
+    level0_len: usize,
+    // Last doc id covered by each entry (level0 entries then level1 entries),
+    // decoded once at construction. Level1 markers are fully validated because
+    // WAND may use them to skip a group; u32::MAX marks malformed entries.
+    entry_doc_up_tos: Arc<[u32]>,
+    // The most recently baked bounds with a stable scorer key. Each query holds
+    // its own Arc in ImpactScoreCache, so replacing this slot for another scorer
+    // cannot change bounds already in use. Scorers without a key never enter the
+    // shared slot. Malformed entries bake to INFINITY so pruning stays safe.
+    last_keyed_bounds: Arc<Mutex<LastKeyedImpactBounds>>,
+}
+
+impl PartialEq for ImpactSkipData {
+    fn eq(&self, other: &Self) -> bool {
+        self.entries == other.entries && self.level0_len == other.level0_len
+    }
+}
+
+#[cfg(test)]
+#[derive(Debug, Clone, Copy)]
+pub struct ImpactScore {
+    pub score: f32,
+    pub entries_scanned: usize,
+}
+
+#[derive(Debug)]
+struct ImpactBounds {
+    per_entry: Box<[f32]>,
+    global: f32,
+}
+
+type LastKeyedImpactBounds = Option<(u64, Arc<ImpactBounds>)>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ImpactBoundsCacheKey {
+    Keyed(u64),
+    QueryLocal,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct ImpactScoreCache {
+    key: Option<ImpactBoundsCacheKey>,
+    bounds: Option<Arc<ImpactBounds>>,
+}
+
+impl ImpactScoreCache {
+    fn bounds<'a, S: Scorer + ?Sized>(
+        &'a mut self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> &'a ImpactBounds {
+        let scorer_key = scorer.doc_weight_cache_key();
+        let cache_key = scorer_key
+            .map(ImpactBoundsCacheKey::Keyed)
+            .unwrap_or(ImpactBoundsCacheKey::QueryLocal);
+        if self.key != Some(cache_key) {
+            self.key = Some(cache_key);
+            self.bounds = None;
+        }
+
+        self.bounds
+            .get_or_insert_with(|| impacts.bounds_for_scorer(scorer, scorer_key))
+    }
+
+    fn entry_score<S: Scorer + ?Sized>(
+        &mut self,
+        impacts: &ImpactSkipData,
+        entry_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+    ) -> f32 {
+        if query_weight <= 0.0 {
+            return 0.0;
+        }
+        query_weight * self.bounds(impacts, scorer).per_entry[entry_idx]
+    }
+}
+
+impl ImpactSkipData {
+    pub fn new(entries: LargeBinaryArray, level0_len: usize) -> Result<Self> {
+        let expected_len = level0_len + level1_len(level0_len);
+        if entries.len() != expected_len {
+            return Err(Error::index(format!(
+                "impact entry count mismatch: got {}, expected {} for {} level0 blocks",
+                entries.len(),
+                expected_len,
+                level0_len
+            )));
+        }
+        let entry_doc_up_tos = (0..entries.len())
+            .map(|entry_idx| {
+                if entries.is_null(entry_idx) {
+                    return u32::MAX;
+                }
+                let bytes = entries.value(entry_idx);
+                let doc_up_to = if entry_idx < level0_len {
+                    decode_level0_entry_doc_up_to(bytes)
+                } else {
+                    decode_entry_doc_up_to(bytes)
+                };
+                doc_up_to.unwrap_or(u32::MAX)
+            })
+            .collect::<Arc<[u32]>>();
+        Ok(Self {
+            entries,
+            level0_len,
+            entry_doc_up_tos,
+            last_keyed_bounds: Arc::new(Mutex::new(None)),
+        })
+    }
+
+    fn keyed_bounds_guard(&self) -> MutexGuard<'_, LastKeyedImpactBounds> {
+        match self.last_keyed_bounds.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        }
+    }
+
+    fn bounds_for_scorer<S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        scorer_key: Option<u64>,
+    ) -> Arc<ImpactBounds> {
+        let Some(scorer_key) = scorer_key else {
+            return Arc::new(self.compute_bounds(scorer));
+        };
+
+        {
+            let cached = self.keyed_bounds_guard();
+            if let Some((cached_key, bounds)) = cached.as_ref()
+                && *cached_key == scorer_key
+            {
+                return bounds.clone();
+            }
+        }
+
+        // Compute outside the mutex. Concurrent misses may duplicate this work,
+        // but only the short publication/check below holds the shared lock.
+        let computed = Arc::new(self.compute_bounds(scorer));
+        let mut cached = self.keyed_bounds_guard();
+        if let Some((cached_key, bounds)) = cached.as_ref()
+            && *cached_key == scorer_key
+        {
+            return bounds.clone();
+        }
+        *cached = Some((scorer_key, computed.clone()));
+        computed
+    }
+
+    fn compute_bounds<S: Scorer + ?Sized>(&self, scorer: &S) -> ImpactBounds {
+        let per_entry = (0..self.entries.len())
+            .map(|entry_idx| {
+                if self.entries.is_null(entry_idx) {
+                    return f32::INFINITY;
+                }
+                let bytes = self.entries.value(entry_idx);
+                let mut max_doc_weight = 0.0_f32;
+                match for_each_entry_pair(bytes, |freq, doc_len| {
+                    max_doc_weight = max_doc_weight.max(scorer.doc_weight(freq, doc_len));
+                }) {
+                    Ok(()) => max_doc_weight,
+                    Err(_) => f32::INFINITY,
+                }
+            })
+            .collect::<Box<[f32]>>();
+        // The level1 entries cover every block, so their max is the list-wide
+        // max doc weight; zero-entry lists fall back to the empty level0 slab.
+        let global = if per_entry.len() > self.level0_len {
+            per_entry[self.level0_len..]
+                .iter()
+                .copied()
+                .fold(0.0_f32, f32::max)
+        } else {
+            per_entry.iter().copied().fold(0.0_f32, f32::max)
+        };
+        ImpactBounds { per_entry, global }
+    }
+
+    /// List-wide max doc weight, from the scorer-specific cached bounds. The
+    /// tightest valid global score bound is `query_weight * this`, matching what
+    /// the non-impact format stores as `max_score` at build time.
+    pub fn global_max_doc_weight_cached<S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        cache.bounds(self, scorer).global
+    }
+
+    /// Cached per-block max doc weights (level0 entries only), for bulk skip
+    /// scans over dead ranges without per-block window bookkeeping.
+    pub(crate) fn level0_doc_weight_bounds_cached<'a, S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        cache: &'a mut ImpactScoreCache,
+    ) -> &'a [f32] {
+        &cache.bounds(self, scorer).per_entry[..self.level0_len]
+    }
+
+    pub fn entries(&self) -> &LargeBinaryArray {
+        &self.entries
+    }
+
+    /// Conservative heap charge for query-independent derived state and one
+    /// shared keyed-bound slab, whether or not that slab has been initialized
+    /// yet. The Arrow impact entries are owned by the enclosing batch and are
+    /// deliberately excluded so packed-group cache accounting counts them once.
+    pub(crate) fn derived_cache_bytes(&self) -> usize {
+        Self::derived_cache_bytes_for_entries(self.entries.len())
+    }
+
+    pub(crate) fn derived_cache_bytes_for_entries(entry_count: usize) -> usize {
+        entry_count * size_of::<u32>()
+            + size_of::<Mutex<LastKeyedImpactBounds>>()
+            + size_of::<ImpactBounds>()
+            + entry_count * size_of::<f32>()
+    }
+
+    #[cfg(test)]
+    pub(crate) fn shares_derived_state_with(&self, other: &Self) -> bool {
+        Arc::ptr_eq(&self.entry_doc_up_tos, &other.entry_doc_up_tos)
+            && Arc::ptr_eq(&self.last_keyed_bounds, &other.last_keyed_bounds)
+    }
+
+    #[cfg(test)]
+    pub fn level0_len(&self) -> usize {
+        self.level0_len
+    }
+
+    #[cfg(test)]
+    pub fn level1_len(&self) -> usize {
+        level1_len(self.level0_len)
+    }
+
+    pub(crate) fn level1_doc_up_to(&self, group_idx: usize) -> Option<u32> {
+        if group_idx >= level1_len(self.level0_len) {
+            return None;
+        }
+        match self.entry_doc_up_tos[self.level0_len + group_idx] {
+            u32::MAX => None,
+            doc_up_to => Some(doc_up_to),
+        }
+    }
+
+    /// Last doc id covered by the level0 entry of `block_idx`, or `None` when
+    /// the entry is missing or malformed.
+    pub(crate) fn level0_doc_up_to(&self, block_idx: usize) -> Option<u32> {
+        if block_idx >= self.level0_len {
+            return None;
+        }
+        match self.entry_doc_up_tos[block_idx] {
+            u32::MAX => None,
+            doc_up_to => Some(doc_up_to),
+        }
+    }
+
+    pub fn level0_score_cached<S: Scorer + ?Sized>(
+        &self,
+        block_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        if block_idx >= self.level0_len {
+            return 0.0;
+        }
+        cache.entry_score(self, block_idx, query_weight, scorer)
+    }
+
+    /// Max score of the docs covered by the level1 entry of `group_idx`,
+    /// answered from the scorer-specific cached bounds slab.
+    pub(crate) fn level1_score_cached<S: Scorer + ?Sized>(
+        &self,
+        group_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        if group_idx >= level1_len(self.level0_len) {
+            return 0.0;
+        }
+        cache.entry_score(self, self.level0_len + group_idx, query_weight, scorer)
+    }
+
+    #[cfg(test)]
+    pub fn max_score_up_to_cached<S>(
+        &self,
+        start_block_idx: usize,
+        up_to: u64,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> ImpactScore
+    where
+        S: Scorer + ?Sized,
+    {
+        let mut block_idx = start_block_idx;
+        let mut max_score = 0.0_f32;
+        let mut entries_scanned = 0usize;
+
+        while block_idx < self.level0_len {
+            let group_idx = block_idx / IMPACT_LEVEL1_BLOCKS;
+            let group_start = group_idx * IMPACT_LEVEL1_BLOCKS;
+            let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(self.level0_len);
+            if block_idx == group_start {
+                let level1_entry_idx = self.level0_len + group_idx;
+                match self.entry_doc_up_tos[level1_entry_idx] {
+                    u32::MAX => {
+                        return ImpactScore {
+                            score: f32::INFINITY,
+                            entries_scanned: entries_scanned + 1,
+                        };
+                    }
+                    doc_up_to if u64::from(doc_up_to) <= up_to => {
+                        max_score = max_score.max(cache.entry_score(
+                            self,
+                            level1_entry_idx,
+                            query_weight,
+                            scorer,
+                        ));
+                        entries_scanned += 1;
+                        block_idx = group_end;
+                        continue;
+                    }
+                    _ => {}
+                }
+            }
+
+            max_score = max_score.max(cache.entry_score(self, block_idx, query_weight, scorer));
+            entries_scanned += 1;
+            match self.entry_doc_up_tos[block_idx] {
+                u32::MAX => {
+                    return ImpactScore {
+                        score: f32::INFINITY,
+                        entries_scanned,
+                    };
+                }
+                doc_up_to if u64::from(doc_up_to) >= up_to => break,
+                _ => {}
+            }
+            block_idx += 1;
+        }
+
+        ImpactScore {
+            score: max_score,
+            entries_scanned,
+        }
+    }
+}
+
+pub struct ImpactSkipDataBuilder {
+    entries: LargeBinaryBuilder,
+    level0_len: usize,
+    level1_entries: Vec<Vec<u8>>,
+    level1_docs: Vec<(u32, u32, u32)>,
+}
+
+impl ImpactSkipDataBuilder {
+    pub fn with_capacity(level0_blocks: usize, block_size: usize) -> Self {
+        Self {
+            entries: LargeBinaryBuilder::with_capacity(
+                level0_blocks + level1_len(level0_blocks),
+                0,
+            ),
+            level0_len: 0,
+            level1_entries: Vec::with_capacity(level1_len(level0_blocks)),
+            level1_docs: Vec::with_capacity(IMPACT_LEVEL1_BLOCKS * block_size),
+        }
+    }
+
+    pub fn append_block(&mut self, docs: &[(u32, u32, u32)]) -> Result<()> {
+        let bytes = encode_impact_entry(docs)?;
+        self.entries.append_value(bytes.as_slice());
+        self.level0_len += 1;
+        self.level1_docs.extend_from_slice(docs);
+        if self.level0_len.is_multiple_of(IMPACT_LEVEL1_BLOCKS) {
+            self.flush_level1()?;
+        }
+        Ok(())
+    }
+
+    pub fn finish(mut self) -> Result<ImpactSkipData> {
+        if !self.level1_docs.is_empty() {
+            self.flush_level1()?;
+        }
+        for entry in self.level1_entries {
+            self.entries.append_value(entry.as_slice());
+        }
+        ImpactSkipData::new(self.entries.finish(), self.level0_len)
+    }
+
+    fn flush_level1(&mut self) -> Result<()> {
+        let bytes = encode_impact_entry(self.level1_docs.as_slice())?;
+        self.level1_entries.push(bytes);
+        self.level1_docs.clear();
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+pub fn build_impact_skip_data(blocks: &[Vec<(u32, u32, u32)>]) -> Result<ImpactSkipData> {
+    let block_size = blocks.iter().map(Vec::len).max().unwrap_or(0).max(1);
+    let mut builder = ImpactSkipDataBuilder::with_capacity(blocks.len(), block_size);
+    for block in blocks {
+        builder.append_block(block)?;
+    }
+    builder.finish()
+}
+
+fn encode_impact_entry(docs: &[(u32, u32, u32)]) -> Result<Vec<u8>> {
+    if docs.is_empty() {
+        return Err(Error::index(
+            "cannot encode an empty impact entry".to_owned(),
+        ));
+    }
+    let doc_up_to = docs
+        .last()
+        .map(|(doc_id, _, _)| *doc_id)
+        .expect("non-empty impact entry was validated above");
+    let frontier = quantized_impact_frontier(docs);
+    let pair_count = u32::try_from(frontier.len()).map_err(|_| {
+        Error::index("impact frontier too large to encode as u32 pair count".to_string())
+    })?;
+    let mut bytes = Vec::with_capacity(5 + frontier.len() * 2);
+    super::encoding::encode_varint_u32(&mut bytes, doc_up_to);
+    super::encoding::encode_varint_u32(&mut bytes, pair_count);
+    let mut previous_freq = 0u32;
+    let mut previous_norm = 0u8;
+    for (pair_idx, (freq, norm)) in frontier.into_iter().enumerate() {
+        let freq_delta_minus_one = freq
+            .checked_sub(previous_freq)
+            .and_then(|delta| delta.checked_sub(1))
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "impact frequencies must be positive and strictly increasing: previous={previous_freq}, current={freq}"
+                ))
+            })?;
+        let norm_delta = norm.checked_sub(previous_norm).ok_or_else(|| {
+            Error::index(format!(
+                "impact norms must be non-decreasing: previous={previous_norm}, current={norm}"
+            ))
+        })?;
+        if pair_idx > 0 && norm_delta == 0 {
+            return Err(Error::index(format!(
+                "impact norms must be strictly increasing after quantization: norm={norm}"
+            )));
+        }
+
+        let has_explicit_norm_delta = norm_delta != 1;
+        let packed_freq_delta =
+            (u64::from(freq_delta_minus_one) << 1) | u64::from(has_explicit_norm_delta);
+        encode_varint_u64(&mut bytes, packed_freq_delta);
+        if has_explicit_norm_delta {
+            bytes.push(norm_delta);
+        }
+        previous_freq = freq;
+        previous_norm = norm;
+    }
+    Ok(bytes)
+}
+
+fn decode_entry_doc_up_to(bytes: &[u8]) -> Result<u32> {
+    let mut offset = 0usize;
+    let doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    // Level-1 doc ids drive whole-group skips, so only publish a doc id after
+    // validating the complete entry. A truncated entry may still have a valid
+    // first varint.
+    for_each_entry_pair(bytes, |_, _| {})?;
+    Ok(doc_up_to)
+}
+
+fn decode_level0_entry_doc_up_to(bytes: &[u8]) -> Result<u32> {
+    // A malformed level0 frontier bakes an INFINITY score before this marker
+    // can terminate a range scan, so avoid parsing every frontier twice on the
+    // query-load path. Level1 entries are fully validated.
+    let mut offset = 0usize;
+    super::encoding::decode_varint_u32(bytes, &mut offset)
+}
+
+/// Walk an entry's (freq, doc_len) frontier pairs, validating the layout.
+fn for_each_entry_pair(bytes: &[u8], mut visit: impl FnMut(u32, u32)) -> Result<()> {
+    let mut offset = 0usize;
+    let _doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    let pair_count = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    if pair_count == 0 {
+        return Err(Error::index(
+            "impact entry must contain at least one frontier pair".to_owned(),
+        ));
+    }
+
+    let mut previous_freq = 0u32;
+    let mut previous_norm = 0u16;
+    for pair_idx in 0..pair_count {
+        let packed_freq_delta = decode_varint_u64(bytes, &mut offset)?;
+        let freq_delta_minus_one = u32::try_from(packed_freq_delta >> 1)
+            .map_err(|_| Error::index("impact freq delta exceeds u32".to_owned()))?;
+        let freq_delta = freq_delta_minus_one
+            .checked_add(1)
+            .ok_or_else(|| Error::index("impact freq delta overflow".to_owned()))?;
+        let freq = previous_freq
+            .checked_add(freq_delta)
+            .ok_or_else(|| Error::index("impact frequency overflow".to_owned()))?;
+
+        let has_explicit_norm_delta = packed_freq_delta & 1 != 0;
+        let norm_delta = if has_explicit_norm_delta {
+            let norm_delta = bytes.get(offset).copied().ok_or_else(|| {
+                Error::index("unexpected EOF while decoding impact norm delta".to_owned())
+            })?;
+            offset += 1;
+            norm_delta
+        } else {
+            1
+        };
+        if pair_idx > 0 && norm_delta == 0 {
+            return Err(Error::index(
+                "impact norms must be strictly increasing".to_owned(),
+            ));
+        }
+
+        let norm = previous_norm
+            .checked_add(u16::from(norm_delta))
+            .filter(|norm| *norm <= u16::from(u8::MAX))
+            .ok_or_else(|| Error::index("impact norm delta overflow".to_owned()))?;
+        let norm = norm as u8;
+        visit(freq, super::index::dequantize_doc_length(norm));
+        previous_freq = freq;
+        previous_norm = u16::from(norm);
+    }
+    if offset != bytes.len() {
+        return Err(Error::index(format!(
+            "impact entry has {} trailing bytes",
+            bytes.len() - offset
+        )));
+    }
+    Ok(())
+}
+
+#[inline]
+fn encode_varint_u64(dst: &mut Vec<u8>, mut value: u64) {
+    while value >= 0x80 {
+        dst.push((value as u8) | 0x80);
+        value >>= 7;
+    }
+    dst.push(value as u8);
+}
+
+#[inline]
+fn decode_varint_u64(src: &[u8], offset: &mut usize) -> Result<u64> {
+    let mut value = 0u64;
+    let mut shift = 0u32;
+    while *offset < src.len() {
+        let byte = src[*offset];
+        *offset += 1;
+        if shift == 63 && byte & 0xFE != 0 {
+            return Err(Error::index(
+                "invalid u64 varint in impact entry".to_owned(),
+            ));
+        }
+        value |= u64::from(byte & 0x7F) << shift;
+        if byte & 0x80 == 0 {
+            return Ok(value);
+        }
+        shift += 7;
+        if shift > 63 {
+            return Err(Error::index(
+                "invalid u64 varint in impact entry".to_owned(),
+            ));
+        }
+    }
+    Err(Error::index(
+        "unexpected EOF while decoding impact entry".to_owned(),
+    ))
+}
+
+fn quantized_impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u8)> {
+    let raw_frontier = impact_frontier(docs);
+    let mut frontier: Vec<(u32, u8)> = Vec::with_capacity(raw_frontier.len());
+    for (freq, doc_len) in raw_frontier {
+        let norm = super::index::quantize_doc_length(doc_len);
+        match frontier.last_mut() {
+            Some((last_freq, last_norm)) if *last_norm == norm => {
+                // At the same quantized norm, the larger frequency dominates.
+                *last_freq = freq;
+            }
+            Some((_, last_norm)) => {
+                debug_assert!(
+                    *last_norm < norm,
+                    "raw impact frontier document lengths must be increasing"
+                );
+                frontier.push((freq, norm));
+            }
+            None => frontier.push((freq, norm)),
+        }
+    }
+    frontier
+}
+
+fn impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> {
+    let max_freq = docs.iter().map(|(_, freq, _)| *freq).max().unwrap_or(0) as usize;
+    if max_freq <= SMALL_FRONTIER_FREQ_LIMIT {
+        return impact_frontier_small_freq(docs, max_freq);
+    }
+
+    impact_frontier_sparse_freq(docs)
+}
+
+fn impact_frontier_small_freq(docs: &[(u32, u32, u32)], max_freq: usize) -> Vec<(u32, u32)> {
+    let mut min_doc_len_by_freq = [u32::MAX; SMALL_FRONTIER_FREQ_LIMIT + 1];
+    for (_, freq, doc_len) in docs {
+        min_doc_len_by_freq[*freq as usize] = min_doc_len_by_freq[*freq as usize].min(*doc_len);
+    }
+
+    let min_doc_lens = min_doc_len_by_freq[..=max_freq]
+        .iter()
+        .enumerate()
+        .filter_map(|(freq, doc_len)| (*doc_len != u32::MAX).then_some((freq as u32, *doc_len)))
+        .collect::<Vec<_>>();
+    frontier_from_min_doc_lens(min_doc_lens)
+}
+
+fn impact_frontier_sparse_freq(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> {
+    let mut pairs = docs
+        .iter()
+        .map(|(_, freq, doc_len)| (*freq, *doc_len))
+        .collect::<Vec<_>>();
+    pairs.sort_unstable_by_key(|(freq, _)| *freq);
+
+    let mut min_doc_lens: Vec<(u32, u32)> = Vec::with_capacity(pairs.len());
+    for (freq, doc_len) in pairs {
+        match min_doc_lens.last_mut() {
+            Some((last_freq, last_doc_len)) if *last_freq == freq => {
+                *last_doc_len = (*last_doc_len).min(doc_len);
+            }
+            _ => min_doc_lens.push((freq, doc_len)),
+        }
+    }
+
+    frontier_from_min_doc_lens(min_doc_lens)
+}
+
+fn frontier_from_min_doc_lens(min_doc_lens: Vec<(u32, u32)>) -> Vec<(u32, u32)> {
+    let mut best_doc_len = u32::MAX;
+    let mut frontier = Vec::with_capacity(min_doc_lens.len());
+    for (freq, doc_len) in min_doc_lens.into_iter().rev() {
+        if doc_len < best_doc_len {
+            frontier.push((freq, doc_len));
+            best_doc_len = doc_len;
+        }
+    }
+    frontier.reverse();
+    frontier
+}
+
+fn level1_len(level0_len: usize) -> usize {
+    level0_len.div_ceil(IMPACT_LEVEL1_BLOCKS)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer};
+
+    use super::*;
+    use crate::scalar::inverted::scorer::{MemBM25Scorer, Scorer};
+
+    struct KeyedCountingScorer {
+        key: u64,
+        calls: Arc<AtomicUsize>,
+    }
+
+    impl Scorer for KeyedCountingScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+            self.calls.fetch_add(1, Ordering::Relaxed);
+            freq as f32 / doc_tokens as f32
+        }
+
+        fn doc_weight_cache_key(&self) -> Option<u64> {
+            Some(self.key)
+        }
+    }
+
+    #[test]
+    fn impact_entry_frontier_drops_dominated_pairs() {
+        let docs = vec![(0, 1, 10), (1, 1, 8), (2, 2, 9), (3, 3, 20)];
+        assert_eq!(impact_frontier(&docs), vec![(1, 8), (2, 9), (3, 20)]);
+    }
+
+    #[test]
+    fn impact_entry_frontier_handles_sparse_large_frequencies() {
+        let docs = vec![
+            (0, 1, 100),
+            (1, 1, 80),
+            (2, 512, 90),
+            (3, 1_000, 120),
+            (4, 1_000, 110),
+        ];
+        assert_eq!(
+            impact_frontier(&docs),
+            vec![(1, 80), (512, 90), (1_000, 110)]
+        );
+    }
+
+    #[test]
+    fn quantized_impact_frontier_drops_equal_norms() {
+        let docs = vec![(0, 1, 16), (1, 2, 17), (2, 3, 24)];
+        assert_eq!(
+            quantized_impact_frontier(&docs),
+            vec![
+                (2, super::super::index::quantize_doc_length(17)),
+                (3, super::super::index::quantize_doc_length(24)),
+            ]
+        );
+    }
+
+    #[test]
+    fn impact_max_score_can_use_level1_entry() {
+        let blocks = (0..40)
+            .map(|block| vec![(block as u32, 1 + block as u32 % 3, 10)])
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        assert_eq!(impacts.level0_len(), 40);
+        assert_eq!(impacts.level1_len(), 2);
+        let scorer = MemBM25Scorer::new(400, 40, HashMap::from([(String::from("token"), 40usize)]));
+        let mut cache = ImpactScoreCache::default();
+        let score = impacts.max_score_up_to_cached(0, 31, 1.0, &scorer, &mut cache);
+        assert!(score.entries_scanned < IMPACT_LEVEL1_BLOCKS);
+        assert!(score.score > 0.0);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_reports_full_and_partial_groups() {
+        let blocks = (0..40)
+            .map(|block| vec![(block as u32, 1, 10)])
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+
+        assert_eq!(
+            impacts.level1_doc_up_to(0),
+            Some((IMPACT_LEVEL1_BLOCKS - 1) as u32)
+        );
+        assert_eq!(impacts.level1_doc_up_to(1), Some(39));
+        assert_eq!(impacts.level1_doc_up_to(2), None);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_returns_none_for_malformed_entry() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let malformed_level1 = vec![1, 2, 3];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(malformed_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_validates_complete_entry() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        // A complete first varint is not enough: the pair count and frontier
+        // are required before this doc id can safely drive a group skip.
+        let truncated_level1 = [31_u8];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(truncated_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+    }
+
+    #[test]
+    fn empty_impact_frontiers_are_malformed_bounds() {
+        let scorer = MemBM25Scorer::new(10, 1, HashMap::new());
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let empty_level1 = [0, 0];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(empty_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+        let mut cache = ImpactScoreCache::default();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+        assert!(
+            impacts
+                .global_max_doc_weight_cached(&scorer, &mut cache)
+                .is_infinite()
+        );
+        assert!(
+            ImpactSkipDataBuilder::with_capacity(1, 128)
+                .append_block(&[])
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn null_impact_entry_is_an_infinite_bound_even_with_hidden_bytes() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let level1 = encode_impact_entry(&[(0, 2, 8)]).unwrap();
+        let mut values = level0.clone();
+        values.extend_from_slice(&level1);
+        let entries = LargeBinaryArray::new(
+            OffsetBuffer::new(ScalarBuffer::from(vec![
+                0_i64,
+                level0.len() as i64,
+                values.len() as i64,
+            ])),
+            Buffer::from_vec(values),
+            Some(NullBuffer::from(vec![true, false])),
+        );
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+        let scorer = MemBM25Scorer::new(10, 1, HashMap::new());
+        let mut cache = ImpactScoreCache::default();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+        assert!(
+            impacts
+                .global_max_doc_weight_cached(&scorer, &mut cache)
+                .is_infinite()
+        );
+    }
+
+    #[test]
+    fn impact_bounds_follow_changed_bm25_average_doc_length() {
+        let impacts = build_impact_skip_data(&[vec![(0, 1, 100)]]).unwrap();
+        let low_avgdl = MemBM25Scorer::new(1, 1, HashMap::new());
+        let high_avgdl = MemBM25Scorer::new(100, 1, HashMap::new());
+        let mut low_cache = ImpactScoreCache::default();
+        let mut high_cache = ImpactScoreCache::default();
+
+        let low_bound = impacts.global_max_doc_weight_cached(&low_avgdl, &mut low_cache);
+        let high_bound = impacts.global_max_doc_weight_cached(&high_avgdl, &mut high_cache);
+        let quantized_doc_length = super::super::index::dequantize_doc_length(
+            super::super::index::quantize_doc_length(100),
+        );
+
+        assert!((low_bound - low_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6);
+        assert!((high_bound - high_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6);
+        assert!(low_bound >= low_avgdl.doc_weight(1, 100));
+        assert!(high_bound >= high_avgdl.doc_weight(1, 100));
+        assert!(
+            high_bound > low_bound,
+            "larger avgdl must recompute a larger bound: low={low_bound}, high={high_bound}"
+        );
+    }
+
+    #[test]
+    fn impact_bounds_reuse_same_scorer_key_across_queries() {
+        let impacts = build_impact_skip_data(&[vec![(0, 2, 10)]]).unwrap();
+        let cloned = impacts.clone();
+        assert!(impacts.shares_derived_state_with(&cloned));
+        let calls = Arc::new(AtomicUsize::new(0));
+        let scorer = KeyedCountingScorer {
+            key: 7,
+            calls: calls.clone(),
+        };
+        let mut first_query = ImpactScoreCache::default();
+        let mut second_query = ImpactScoreCache::default();
+
+        let first = impacts.global_max_doc_weight_cached(&scorer, &mut first_query);
+        let baked_calls = calls.load(Ordering::Relaxed);
+        assert!(baked_calls > 0);
+        let second = cloned.global_max_doc_weight_cached(&scorer, &mut second_query);
+
+        assert_eq!(second, first);
+        assert_eq!(
+            calls.load(Ordering::Relaxed),
+            baked_calls,
+            "the same keyed bounds should be shared across query caches"
+        );
+    }
+
+    #[test]
+    fn malformed_unscanned_entry_does_not_poison_range_score() {
+        let level0_0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let malformed_level0_1 = vec![1, 2, 3];
+        let level1 = encode_impact_entry(&[(0, 1, 10), (1, 1, 10)]).unwrap();
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0_0.as_slice()),
+            Some(malformed_level0_1.as_slice()),
+            Some(level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 2).unwrap();
+        let scorer = MemBM25Scorer::new(10, 10, HashMap::from([(String::from("token"), 2usize)]));
+        let mut cache = ImpactScoreCache::default();
+
+        let score = impacts.max_score_up_to_cached(0, 0, 1.0, &scorer, &mut cache);
+        assert!(score.score.is_finite());
+        assert_eq!(score.entries_scanned, 1);
+
+        assert_eq!(
+            impacts.level0_score_cached(1, 1.0, &scorer, &mut cache),
+            f32::INFINITY
+        );
+    }
+
+    #[test]
+    fn impact_entries_store_quantized_norm_deltas() {
+        let docs = vec![(7, 1, 1), (9, 2, 2), (12, 3, 5)];
+        let encoded = encode_impact_entry(&docs).unwrap();
+
+        // doc_up_to=12, pair_count=3, two implicit +1 norm deltas, then an
+        // explicit +3 norm delta folded behind the frequency varint's low bit.
+        assert_eq!(encoded, vec![12, 3, 0, 0, 1, 3]);
+
+        let mut pairs = Vec::new();
+        for_each_entry_pair(&encoded, |freq, doc_len| pairs.push((freq, doc_len))).unwrap();
+        assert_eq!(pairs, vec![(1, 1), (2, 2), (3, 5)]);
+    }
+
+    #[test]
+    fn malformed_norm_deltas_are_rejected() {
+        // doc_up_to=0, pair_count=1, and the pair flag promises a norm byte
+        // that is not present.
+        let truncated = [0, 1, 1];
+        let error = for_each_entry_pair(&truncated, |_, _| {}).unwrap_err();
+        assert!(matches!(&error, Error::Index { .. }));
+        assert!(error.to_string().contains("impact norm delta"));
+
+        // The first pair reaches norm 255, so an implicit +1 on the second
+        // pair must fail instead of wrapping back to zero.
+        let overflowing = [0, 2, 1, 255, 0];
+        let error = for_each_entry_pair(&overflowing, |_, _| {}).unwrap_err();
+        assert!(matches!(&error, Error::Index { .. }));
+        assert!(error.to_string().contains("impact norm delta overflow"));
+    }
+
+    #[test]
+    fn impact_entries_roundtrip_quantized_frontier() {
+        let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80), (4095, 130, 900)];
+        let encoded = encode_impact_entry(&docs).unwrap();
+        assert_eq!(decode_entry_doc_up_to(&encoded).unwrap(), 4095);
+        let mut decoded_pairs = Vec::new();
+        for_each_entry_pair(&encoded, |freq, doc_len| {
+            decoded_pairs.push((freq, doc_len))
+        })
+        .unwrap();
+        let expected_pairs = quantized_impact_frontier(&docs)
+            .into_iter()
+            .map(|(freq, norm)| (freq, super::super::index::dequantize_doc_length(norm)))
+            .collect::<Vec<_>>();
+        assert_eq!(decoded_pairs, expected_pairs);
+        assert!(!decoded_pairs.is_empty());
+
+        // A 256-doc-block skip data goes through the shared codec end to end.
+        let blocks: Vec<Vec<(u32, u32, u32)>> = (0..3)
+            .map(|b| (0..256).map(|i| (b * 256 + i, 1 + i % 5, 10)).collect())
+            .collect();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        assert_eq!(impacts.level1_doc_up_to(0), Some(767));
+        let scorer = MemBM25Scorer::new(400, 768, HashMap::from([(String::from("t"), 768usize)]));
+        let mut cache = ImpactScoreCache::default();
+        assert!(
+            impacts
+                .level0_score_cached(0, 1.0, &scorer, &mut cache)
+                .is_finite()
+        );
+        let level1 = impacts.max_score_up_to_cached(0, 767, 1.0, &scorer, &mut cache);
+        assert!(level1.score.is_finite() && level1.score > 0.0);
+    }
+
+    #[test]
+    fn posting_block_sizes_use_identical_impact_encoding() {
+        let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80)];
+        let mut v2_builder = ImpactSkipDataBuilder::with_capacity(1, 128);
+        v2_builder.append_block(&docs).unwrap();
+        let v2 = v2_builder.finish().unwrap();
+        let mut v3_builder = ImpactSkipDataBuilder::with_capacity(1, 256);
+        v3_builder.append_block(&docs).unwrap();
+        let v3 = v3_builder.finish().unwrap();
+
+        assert_eq!(v2.entries(), v3.entries());
+    }
+
+    #[test]
+    fn impact_upper_bound_covers_real_scores() {
+        let blocks = vec![
+            vec![(0, 1, 100), (3, 2, 40), (7, 4, 80)],
+            vec![(9, 3, 15), (10, 1, 5), (12, 5, 30)],
+            vec![(16, 2, 10), (18, 6, 70), (21, 3, 12)],
+            vec![(24, 1, 4), (28, 7, 100), (30, 2, 8)],
+        ];
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        let scorer = MemBM25Scorer::new(474, 31, HashMap::from([(String::from("token"), 4usize)]));
+        let query_weight = scorer.query_weight("token");
+        let mut cache = ImpactScoreCache::default();
+
+        for start_block_idx in 0..blocks.len() {
+            let up_to = blocks
+                .iter()
+                .skip(start_block_idx)
+                .take(2)
+                .flatten()
+                .map(|(doc_id, _, _)| *doc_id)
+                .max()
+                .unwrap();
+            let upper_bound = impacts.max_score_up_to_cached(
+                start_block_idx,
+                u64::from(up_to),
+                query_weight,
+                &scorer,
+                &mut cache,
+            );
+            let exact_max = blocks
+                .iter()
+                .skip(start_block_idx)
+                .flatten()
+                .take_while(|(doc_id, _, _)| *doc_id <= up_to)
+                .map(|(_, freq, doc_len)| query_weight * scorer.doc_weight(*freq, *doc_len))
+                .fold(0.0_f32, f32::max);
+            assert!(
+                upper_bound.score + 1e-6 >= exact_max,
+                "upper bound {} should cover exact max {} from block {} up to doc {}",
+                upper_bound.score,
+                exact_max,
+                start_block_idx,
+                up_to
+            );
+        }
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index d5a513c7112..368660133ca 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -3,14 +3,14 @@
 
 use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::fmt::{Debug, Display};
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock};
 use std::{
     cmp::{Reverse, min},
     collections::BinaryHeap,
 };
 use std::{
-    collections::{BTreeMap, HashMap, HashSet},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     ops::Range,
     time::Instant,
 };
@@ -18,7 +18,7 @@ use std::{
 use crate::metrics::NoOpMetricsCollector;
 use crate::prefilter::NoFilter;
 use crate::scalar::registry::{TrainingCriteria, TrainingOrdering};
-use arrow::array::{FixedSizeListBuilder, Float32Builder};
+use arrow::array::{FixedSizeListBuilder, Float32Builder, Int32Builder};
 use arrow::datatypes::{self, Float32Type, Int32Type, UInt64Type};
 use arrow::{
     array::{
@@ -51,16 +51,18 @@ use lance_select::{RowAddrMask, RowAddrTreeMap};
 use roaring::RoaringBitmap;
 use std::sync::LazyLock;
 use tokio::{sync::OnceCell, task::spawn_blocking};
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 
-use super::encoding::{PositionBlockBuilder, decode_group_starts};
+use super::encoding::{MAX_POSTING_BLOCK_SIZE, PositionBlockBuilder};
+use super::impact::{IMPACT_LEVEL1_BLOCKS, ImpactSkipData, ImpactSkipDataBuilder};
 use super::iter::PostingListIterator;
 use super::lazy_docset::LazyDocSet;
+use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size};
 use super::{InvertedIndexBuilder, InvertedIndexParams, wand::*};
 use super::{
     builder::{
-        BLOCK_SIZE, PostingGroupAccumulator, PostingGroupConfig, ScoredDoc, doc_file_path,
-        inverted_list_schema_for_version, posting_file_path, token_file_path,
+        BLOCK_SIZE, doc_file_path, inverted_list_schema_for_version_with_block_size_and_impacts,
+        posting_file_path, token_file_path,
     },
     iter::PlainPostingListIterator,
     query::*,
@@ -79,6 +81,7 @@ use crate::scalar::{
     OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery,
     UpdateCriteria,
 };
+use crate::vector::graph::OrderedFloat;
 use crate::{FtsPrewarmOptions, Index};
 use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys};
 use std::str::FromStr;
@@ -86,13 +89,30 @@ use std::str::FromStr;
 // Version 0: Arrow TokenSetFormat (legacy)
 // Version 1: Fst TokenSetFormat with per-doc compressed positions
 // Version 2: Fst TokenSetFormat with shared posting-list position streams.
+// Version 3: Version 2 layout with configurable posting blocks and analyzer metadata.
 pub const INVERTED_INDEX_VERSION_V1: u32 = 1;
 pub const INVERTED_INDEX_VERSION_V2: u32 = 2;
+pub const INVERTED_INDEX_VERSION_V3: u32 = 3;
 pub const TOKENS_FILE: &str = "tokens.lance";
 pub const INVERT_LIST_FILE: &str = "invert.lance";
 pub const DOCS_FILE: &str = "docs.lance";
 pub const METADATA_FILE: &str = "metadata.lance";
 
+/// Partitions searched per cpu-pool task in `bm25_search`. Chunking bounds
+/// the task rate and lets the shared top-k floor propagate between the
+/// partitions a thread scores back-to-back. `LANCE_FTS_SEARCH_CHUNK=1`
+/// restores one-task-per-partition.
+fn fts_search_chunk() -> usize {
+    static CHUNK: LazyLock<usize> = LazyLock::new(|| {
+        std::env::var("LANCE_FTS_SEARCH_CHUNK")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .filter(|&n| n >= 1)
+            .unwrap_or(16)
+    });
+    *CHUNK
+}
+
 pub const TOKEN_COL: &str = "_token";
 pub const TOKEN_ID_COL: &str = "_token_id";
 pub const TOKEN_FST_BYTES_COL: &str = "_token_fst_bytes";
@@ -103,6 +123,7 @@ pub const POSITION_COL: &str = "_position";
 pub const COMPRESSED_POSITION_COL: &str = "_compressed_position";
 pub const POSITION_BLOCK_OFFSET_COL: &str = "_position_block_offset";
 pub const POSTING_COL: &str = "_posting";
+pub const IMPACT_COL: &str = "_impacts";
 pub const MAX_SCORE_COL: &str = "_max_score";
 pub const LENGTH_COL: &str = "_length";
 pub const BLOCK_MAX_SCORE_COL: &str = "_block_max_score";
@@ -110,13 +131,10 @@ pub const NUM_TOKEN_COL: &str = "_num_tokens";
 pub const SCORE_COL: &str = "_score";
 pub const TOKEN_SET_FORMAT_KEY: &str = "token_set_format";
 pub const POSTING_TAIL_CODEC_KEY: &str = "posting_tail_codec";
+pub const FTS_FORMAT_VERSION_KEY: &str = "format_version";
 pub const POSITIONS_LAYOUT_KEY: &str = "positions_layout";
 pub const POSITIONS_CODEC_KEY: &str = "positions_codec";
-/// Schema-metadata key holding the 1-indexed global-buffer id of the
-/// varint-delta-encoded posting-list cache-group boundaries (issue #7040).
-/// Absent on indexes written before grouping was introduced, which fall back
-/// to the per-token cache path.
-pub const POSTING_GROUP_OFFSETS_BUF_KEY: &str = "posting_group_offsets_buf";
+pub const POSTING_BLOCK_SIZE_KEY: &str = "posting_block_size";
 pub const POSTING_TAIL_CODEC_FIXED32_V1: &str = "fixed32_v1";
 pub const POSTING_TAIL_CODEC_VARINT_DELTA_V1: &str = "varint_delta_v1";
 pub const POSITIONS_LAYOUT_SHARED_STREAM_V2: &str = "shared_stream_v2";
@@ -152,7 +170,7 @@ pub fn current_fts_format_version() -> InvertedListFormatVersion {
 }
 
 pub fn max_supported_fts_format_version() -> InvertedListFormatVersion {
-    InvertedListFormatVersion::V2
+    InvertedListFormatVersion::V3
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
@@ -160,6 +178,7 @@ pub enum InvertedListFormatVersion {
     V1,
     #[default]
     V2,
+    V3,
 }
 
 impl InvertedListFormatVersion {
@@ -170,29 +189,50 @@ impl InvertedListFormatVersion {
         }
     }
 
+    pub fn from_posting_tail_codec_and_block_size(
+        codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Result<Self> {
+        validate_block_size(block_size)?;
+        let format_version = match (codec, block_size) {
+            (PostingTailCodec::Fixed32, LEGACY_BLOCK_SIZE) => Self::V1,
+            (PostingTailCodec::VarintDelta, LEGACY_BLOCK_SIZE) => Self::V2,
+            (PostingTailCodec::VarintDelta, 256) => Self::V3,
+            (PostingTailCodec::Fixed32, 256) => {
+                return Err(Error::invalid_input(
+                    "FTS format_version=3 requires the varint-delta posting tail codec".to_string(),
+                ));
+            }
+            _ => unreachable!("validate_block_size limits supported block sizes"),
+        };
+        validate_format_version_block_size(format_version, block_size)?;
+        Ok(format_version)
+    }
+
     pub fn index_version(self) -> u32 {
         match self {
             Self::V1 => INVERTED_INDEX_VERSION_V1,
             Self::V2 => INVERTED_INDEX_VERSION_V2,
+            Self::V3 => INVERTED_INDEX_VERSION_V3,
         }
     }
 
     pub fn posting_tail_codec(self) -> PostingTailCodec {
         match self {
             Self::V1 => PostingTailCodec::Fixed32,
-            Self::V2 => PostingTailCodec::VarintDelta,
+            Self::V2 | Self::V3 => PostingTailCodec::VarintDelta,
         }
     }
 
     pub fn position_codec(self) -> Option<PositionStreamCodec> {
         match self {
             Self::V1 => None,
-            Self::V2 => Some(PositionStreamCodec::PackedDelta),
+            Self::V2 | Self::V3 => Some(PositionStreamCodec::PackedDelta),
         }
     }
 
     pub fn uses_shared_position_stream(self) -> bool {
-        matches!(self, Self::V2)
+        matches!(self, Self::V2 | Self::V3)
     }
 }
 
@@ -203,14 +243,44 @@ impl FromStr for InvertedListFormatVersion {
         match s.trim() {
             "1" | "v1" | "V1" => Ok(Self::V1),
             "2" | "v2" | "V2" => Ok(Self::V2),
+            "3" | "v3" | "V3" => Ok(Self::V3),
             other => Err(Error::index(format!(
-                "unsupported FTS format version {}, expected 1 or 2",
+                "unsupported FTS format version {}, expected 1, 2, or 3",
                 other
             ))),
         }
     }
 }
 
+pub fn default_fts_format_version_for_block_size(
+    block_size: usize,
+) -> Result<InvertedListFormatVersion> {
+    validate_block_size(block_size)?;
+    match block_size {
+        LEGACY_BLOCK_SIZE => Ok(InvertedListFormatVersion::V2),
+        256 => Ok(InvertedListFormatVersion::V3),
+        _ => unreachable!("validate_block_size limits supported block sizes"),
+    }
+}
+
+pub fn validate_format_version_block_size(
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+) -> Result<()> {
+    validate_block_size(block_size)?;
+    match (format_version, block_size) {
+        (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)
+        | (InvertedListFormatVersion::V3, _) => Ok(()),
+        (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, 256) => {
+            Err(Error::invalid_input(format!(
+                "FTS format_version={} is incompatible with block_size=256; use format_version=3",
+                format_version.index_version()
+            )))
+        }
+        _ => unreachable!("validate_block_size limits supported block sizes"),
+    }
+}
+
 #[derive(Debug)]
 struct PartitionCandidates {
     tokens_by_position: Vec<String>,
@@ -218,20 +288,12 @@ struct PartitionCandidates {
     candidates: Vec<DocCandidate>,
 }
 
-impl PartitionCandidates {
-    fn empty() -> Self {
-        Self {
-            tokens_by_position: Vec::new(),
-            grouped_expansions: Vec::new(),
-            candidates: Vec::new(),
-        }
-    }
-}
-
 #[derive(Debug)]
 struct LoadedPostings {
     postings: Vec<PostingIterator>,
     grouped_expansions: Vec<GroupedExpansionTerms>,
+    impact_safe: bool,
+    exact_scoring_required: bool,
 }
 
 impl LoadedPostings {
@@ -239,6 +301,8 @@ impl LoadedPostings {
         Self {
             postings: Vec::new(),
             grouped_expansions: Vec::new(),
+            impact_safe: false,
+            exact_scoring_required: false,
         }
     }
 }
@@ -246,48 +310,7 @@ impl LoadedPostings {
 #[derive(Debug)]
 struct GroupedExpansionTerms {
     position: u32,
-    terms: Vec<ExpansionTermFreqs>,
-}
-
-fn grouped_rescore_wand_limit(
-    limit: Option<usize>,
-    grouped_expansions: &[GroupedExpansionTerms],
-) -> Option<usize> {
-    let limit = limit?;
-    // Grouped fuzzy AND rescoring needs a small candidate cushion because WAND
-    // ranks by the unioned group posting first and the exact expansion IDF later.
-    let expansion_terms = grouped_expansions
-        .iter()
-        .map(|group| group.terms.len())
-        .sum::<usize>()
-        .max(1);
-    Some(limit.saturating_mul(expansion_terms))
-}
-
-#[derive(Debug)]
-struct ExpansionTermFreqs {
-    token: String,
-    freqs_by_posting_doc_id: Vec<(u64, u32)>,
-}
-
-impl ExpansionTermFreqs {
-    fn new(token: String, posting: &PostingList) -> Self {
-        let freqs_by_posting_doc_id = posting
-            .iter()
-            .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq))
-            .collect();
-        Self {
-            token,
-            freqs_by_posting_doc_id,
-        }
-    }
-
-    fn frequency(&self, posting_doc_id: u64) -> Option<u32> {
-        self.freqs_by_posting_doc_id
-            .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id)
-            .ok()
-            .map(|idx| self.freqs_by_posting_doc_id[idx].1)
-    }
+    terms: Arc<[GroupedTermScorer]>,
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Default)]
@@ -372,6 +395,21 @@ pub(super) fn parse_posting_tail_codec(
         .unwrap_or(PostingTailCodec::Fixed32))
 }
 
+pub(super) fn parse_posting_block_size(metadata: &HashMap<String, String>) -> Result<usize> {
+    metadata
+        .get(POSTING_BLOCK_SIZE_KEY)
+        .map(|value| {
+            let block_size = value.parse::<usize>().map_err(|err| {
+                Error::index(format!(
+                    "invalid {POSTING_BLOCK_SIZE_KEY} metadata value {value:?}: {err}"
+                ))
+            })?;
+            validate_block_size(block_size)
+        })
+        .transpose()
+        .map(|block_size| block_size.unwrap_or(LEGACY_BLOCK_SIZE))
+}
+
 impl PositionStreamCodec {
     pub fn as_str(self) -> &'static str {
         match self {
@@ -409,6 +447,26 @@ fn parse_shared_position_codec(metadata: &HashMap<String, String>) -> Result<Pos
 pub(super) fn parse_format_version_from_metadata(
     metadata: &HashMap<String, String>,
 ) -> Result<InvertedListFormatVersion> {
+    if let Some(value) = metadata.get(FTS_FORMAT_VERSION_KEY) {
+        let format_version = InvertedListFormatVersion::from_str(value)?;
+        let block_size = parse_posting_block_size(metadata)?;
+        validate_format_version_block_size(format_version, block_size)?;
+        return Ok(format_version);
+    }
+    let block_size = parse_posting_block_size(metadata)?;
+    if block_size == 256 {
+        if metadata
+            .get(POSTING_TAIL_CODEC_KEY)
+            .map(|_| parse_posting_tail_codec(metadata))
+            .transpose()?
+            .is_some_and(|posting_tail_codec| posting_tail_codec != PostingTailCodec::VarintDelta)
+        {
+            return Err(Error::index(
+                "FTS block_size=256 requires the varint-delta posting tail codec".to_string(),
+            ));
+        }
+        return Ok(InvertedListFormatVersion::V3);
+    }
     if metadata.contains_key(POSITIONS_CODEC_KEY) || metadata.contains_key(POSITIONS_LAYOUT_KEY) {
         return Ok(InvertedListFormatVersion::V2);
     }
@@ -451,44 +509,18 @@ impl DeepSizeOf for InvertedIndex {
     }
 }
 
-/// Resolve any `Pending` candidates that wand emitted via the
-/// deferred-row_id path. After this returns, every entry in
-/// `candidates` carries a real row_id.
-async fn resolve_deferred_candidates(
-    docs: &LazyDocSet,
-    candidates: &mut [DocCandidate],
-) -> Result<()> {
-    let pending: Vec<u32> = candidates
-        .iter()
-        .filter_map(|c| match c.addr {
-            CandidateAddr::Pending(d) => Some(d),
-            CandidateAddr::RowId(_) => None,
-        })
-        .collect();
-    if pending.is_empty() {
-        return Ok(());
-    }
-    let mut iter = docs.resolve_row_ids(&pending).await?.into_iter();
-    for c in candidates {
-        if matches!(c.addr, CandidateAddr::Pending(_)) {
-            let r = iter.next().ok_or_else(|| {
-                Error::internal("resolve_row_ids returned fewer items than requested")
-            })?;
-            c.addr = CandidateAddr::RowId(r);
-        }
-    }
-    Ok(())
-}
-
 impl InvertedIndex {
     fn format_version(&self) -> InvertedListFormatVersion {
         self.format_version
     }
 
     fn index_version(&self) -> u32 {
-        match self.token_set_format {
-            TokenSetFormat::Arrow => 0,
-            TokenSetFormat::Fst => self.format_version().index_version(),
+        match (self.token_set_format, self.format_version()) {
+            (
+                TokenSetFormat::Arrow,
+                InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2,
+            ) => 0,
+            (_, format_version) => format_version.index_version(),
         }
     }
 
@@ -604,8 +636,7 @@ impl InvertedIndex {
         let mut builder = InvertedIndexBuilder::new(first.params.clone()).with_progress(progress);
         builder = builder
             .with_token_set_format(first.token_set_format)
-            .with_format_version(first.format_version())
-            .with_posting_tail_codec(first.posting_tail_codec());
+            .with_format_version(first.format_version());
         let files = builder
             .update_from_segments(new_data, dest_store, segments, old_data_filter)
             .await?;
@@ -630,22 +661,25 @@ impl InvertedIndex {
         query_tokens: &Tokens,
         params: &FtsSearchParams,
     ) -> Result<MemBM25Scorer> {
-        let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?;
-        let mut terms: Vec<String> = Vec::new();
-        let mut seen = HashSet::new();
         if matches!(params.fuzziness, Some(n) if n != 0) {
             let expanded = self.expand_fuzzy_tokens(query_tokens, params)?;
-            for idx in 0..expanded.len() {
-                let token = expanded.get_token(idx);
-                if seen.insert(token.to_string()) {
-                    terms.push(token.to_string());
-                }
-            }
+            self.bm25_scorer_for_final_tokens(&expanded).await
         } else {
-            for token in query_tokens {
-                if seen.insert(token.to_string()) {
-                    terms.push(token.to_string());
-                }
+            self.bm25_scorer_for_final_tokens(query_tokens).await
+        }
+    }
+
+    /// Scorer for a token list that needs no further fuzzy expansion: dedup
+    /// the terms and pull their document frequencies. `bm25_search` calls
+    /// this with the tokens it already expanded, so the expansion runs once
+    /// per query rather than once for the scorer and once per partition.
+    async fn bm25_scorer_for_final_tokens(&self, tokens: &Tokens) -> Result<MemBM25Scorer> {
+        let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?;
+        let mut terms: Vec<String> = Vec::new();
+        let mut seen = HashSet::new();
+        for token in tokens {
+            if seen.insert(token.to_string()) {
+                terms.push(token.to_string());
             }
         }
         let mut token_docs = HashMap::with_capacity(terms.len());
@@ -717,17 +751,46 @@ impl InvertedIndex {
     }
 
     /// Expand fuzzy query tokens against all partitions in this segment.
+    ///
+    /// `params.max_expansions` caps the whole query's expansion, not any
+    /// single partition's: for each query token the per-partition candidates
+    /// (each streamed in FST key order) merge into one lexicographically
+    /// ordered set, and the remaining budget takes a prefix of it. The
+    /// selected terms are a pure function of the segment's vocabulary, so
+    /// splitting the same corpus into more partitions cannot change which
+    /// terms a fuzzy query matches.
     pub fn expand_fuzzy_tokens(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result<Tokens> {
         let mut expanded_tokens = Vec::new();
         let mut expanded_positions = Vec::new();
         let mut seen = HashSet::new();
-        for partition in &self.partitions {
-            let expanded = partition.expand_fuzzy(tokens, params)?;
-            for idx in 0..expanded.len() {
-                let token = expanded.get_token(idx);
-                let position = expanded.position(idx);
-                if seen.insert((token.to_string(), position)) {
-                    expanded_tokens.push(token.to_string());
+        for token_idx in 0..tokens.len() {
+            let remaining = params.max_expansions.saturating_sub(expanded_tokens.len());
+            if remaining == 0 {
+                break;
+            }
+            let token = tokens.get_token(token_idx);
+            let position = tokens.position(token_idx);
+            // Each partition contributes at most its `remaining`
+            // lexicographically smallest candidates, so the global
+            // lex-smallest `remaining` selection below is unaffected by the
+            // per-partition truncation.
+            let mut candidates = BTreeSet::new();
+            let base_prefix_len = tokens.token_type().prefix_len(token) as u32;
+            for partition in &self.partitions {
+                partition.collect_fuzzy_candidates(
+                    token,
+                    base_prefix_len,
+                    params,
+                    remaining,
+                    &mut candidates,
+                )?;
+            }
+            for candidate in candidates {
+                if expanded_tokens.len() >= params.max_expansions {
+                    break;
+                }
+                if seen.insert((candidate.clone(), position)) {
+                    expanded_tokens.push(candidate);
                     expanded_positions.push(position);
                 }
             }
@@ -753,49 +816,94 @@ impl InvertedIndex {
         metrics: Arc<dyn MetricsCollector>,
         base_scorer: Option<&MemBM25Scorer>,
     ) -> Result<(Vec<u64>, Vec<f32>)> {
+        // Fuzzy expansion runs once here, with the global `max_expansions`
+        // budget, instead of once per partition: partitions receive the
+        // final token list, so the matched terms cannot depend on how the
+        // corpus happens to be partitioned.
+        let tokens = if matches!(params.fuzziness, Some(n) if n != 0) {
+            let expanded = Arc::new(self.expand_fuzzy_tokens(tokens.as_ref(), params.as_ref())?);
+            if operator == Operator::And || params.phrase_slop.is_some() {
+                // AND/phrase semantics require every original token position
+                // to keep at least one expansion; a position that expands to
+                // nothing anywhere in the segment can never be matched.
+                let surviving = (0..expanded.len())
+                    .map(|idx| expanded.position(idx))
+                    .collect::<HashSet<_>>();
+                if (0..tokens.len()).any(|idx| !surviving.contains(&tokens.position(idx))) {
+                    return Ok((Vec::new(), Vec::new()));
+                }
+            }
+            expanded
+        } else {
+            tokens
+        };
+
         // The wand only consults `scorer.doc_weight`, which is metadata-free.
         // The outer aggregation below consults `scorer.query_weight`, which
         // hits per-token `posting_len`; building a `MemBM25Scorer` with
         // precomputed per-term IDFs avoids the v2 bulk metadata pull.
         let local_scorer;
-        let scorer: &dyn Scorer = if let Some(base_scorer) = base_scorer {
+        let scorer: &MemBM25Scorer = if let Some(base_scorer) = base_scorer {
             base_scorer
         } else {
-            local_scorer = self
-                .bm25_base_scorer(tokens.as_ref(), params.as_ref())
-                .await?;
+            local_scorer = self.bm25_scorer_for_final_tokens(tokens.as_ref()).await?;
             &local_scorer
         };
+        let impact_scorer = Arc::new(scorer.clone());
 
         let limit = params.limit.unwrap_or(usize::MAX);
         if limit == 0 {
             return Ok((Vec::new(), Vec::new()));
         }
 
+        /// Global-heap entry carrying the score plus where its row_id comes
+        /// from. `Pending` candidates keep (slot, doc_id) through the top-k
+        /// so only the final survivors pay row_id resolution.
+        struct MergedCandidate {
+            score: OrderedFloat,
+            /// Index into the per-query partition list for `Pending` addresses.
+            slot: u32,
+            addr: CandidateAddr,
+        }
+
+        impl PartialEq for MergedCandidate {
+            fn eq(&self, other: &Self) -> bool {
+                self.score == other.score
+            }
+        }
+
+        impl Eq for MergedCandidate {}
+
+        impl PartialOrd for MergedCandidate {
+            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+                Some(self.cmp(other))
+            }
+        }
+
+        impl Ord for MergedCandidate {
+            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+                self.score.cmp(&other.score)
+            }
+        }
+
         fn push_scored_candidate(
-            candidates: &mut BinaryHeap<Reverse<ScoredDoc>>,
+            candidates: &mut BinaryHeap<Reverse<MergedCandidate>>,
             limit: usize,
+            slot: u32,
             addr: CandidateAddr,
             score: f32,
-        ) -> Result<()> {
-            // resolve_deferred_candidates ran upstream, so every candidate
-            // carries a real row_id at this point.
-            let row_id = match addr {
-                CandidateAddr::RowId(r) => r,
-                CandidateAddr::Pending(_) => {
-                    return Err(Error::internal(
-                        "bm25_search post-condition: deferred candidate left unresolved",
-                    ));
-                }
+        ) {
+            let candidate = MergedCandidate {
+                score: OrderedFloat(score),
+                slot,
+                addr,
             };
-
             if candidates.len() < limit {
-                candidates.push(Reverse(ScoredDoc::new(row_id, score)));
+                candidates.push(Reverse(candidate));
             } else if candidates.peek().unwrap().0.score.0 < score {
                 candidates.pop();
-                candidates.push(Reverse(ScoredDoc::new(row_id, score)));
+                candidates.push(Reverse(candidate));
             }
-            Ok(())
         }
 
         let mask = prefilter.mask();
@@ -804,96 +912,156 @@ impl InvertedIndex {
         // Shared top-k floor across this query's partitions. Seeded to -inf so
         // the first real score wins; each partition publishes its local k-th
         // and prunes against the running global k-th (a lower bound on the true
-        // global k-th — see `Wand::shared_threshold`).
-        let shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+        // global k-th - see `Wand::shared_threshold`). Only sound for the
+        // impact (global) scorer, whose scores are comparable across
+        // partitions; legacy BM25 scores are partition-local scale, so those
+        // partitions each get a private floor below.
+        let impact_shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+        // Partitions are processed in chunks: each chunk loads its postings
+        // and scoring DocSets asynchronously, then ONE cpu-pool task searches
+        // the whole chunk sequentially. Chunks pipeline against each other
+        // through buffer_unordered (chunk i searches while chunk i+1 loads).
+        // Chunking bounds the cpu-pool task rate (vs one tiny task per
+        // partition) and lets the shared top-k floor propagate immediately
+        // between partitions searched on the same thread.
         let parts = self
             .partitions
-            .iter()
-            .map(|part| {
-                let part = part.clone();
+            .chunks(fts_search_chunk())
+            .map(|chunk| {
+                let chunk = chunk.to_vec();
                 let tokens = tokens.clone();
                 let params = params.clone();
                 let mask = mask.clone();
                 let metrics = metrics.clone();
-                let shared_threshold = shared_threshold.clone();
+                let impact_scorer = impact_scorer.clone();
+                let impact_shared_threshold = impact_shared_threshold.clone();
                 async move {
-                    let loaded_postings = part
-                        .load_posting_lists(
-                            tokens.as_ref(),
-                            params.as_ref(),
-                            operator,
-                            metrics.as_ref(),
-                        )
-                        .await?;
-                    let LoadedPostings {
-                        postings,
-                        grouped_expansions,
-                    } = loaded_postings;
-                    if postings.is_empty() {
-                        // No hits in this partition; its DocSet stays
-                        // unloaded, so we never pay the per-doc
-                        // row_id/num_tokens download for it.
-                        return Result::Ok(PartitionCandidates::empty());
-                    }
-                    let docs_for_wand = part.docs.docs_for_wand(mask.as_ref()).await?;
-                    let max_position = postings
-                        .iter()
-                        .map(|posting| posting.term_index() as usize)
-                        .max()
-                        .unwrap_or_default();
-                    let mut tokens_by_position = vec![String::new(); max_position + 1];
-                    for posting in &postings {
-                        let idx = posting.term_index() as usize;
-                        tokens_by_position[idx] = posting.token().to_owned();
+                    // Load the chunk's partitions concurrently; only the
+                    // search below is batched onto one cpu task.
+                    let loads = chunk.into_iter().map(|part| {
+                        let tokens = tokens.clone();
+                        let params = params.clone();
+                        let mask = mask.clone();
+                        let metrics = metrics.clone();
+                        let impact_scorer = impact_scorer.clone();
+                        let impact_shared_threshold = impact_shared_threshold.clone();
+                        async move {
+                            let loaded_postings = part
+                                .load_posting_lists(
+                                    tokens.as_ref(),
+                                    params.as_ref(),
+                                    operator,
+                                    impact_scorer.as_ref(),
+                                    metrics.as_ref(),
+                                )
+                                .await?;
+                            let LoadedPostings {
+                                postings,
+                                grouped_expansions,
+                                impact_safe,
+                                exact_scoring_required,
+                            } = loaded_postings;
+                            if postings.is_empty() {
+                                // No hits in this partition; its DocSet stays
+                                // unloaded, so we never pay the per-doc
+                                // row_id/num_tokens download for it.
+                                return Result::Ok(None);
+                            }
+                            let docs_for_wand =
+                                part.docs.docs_for_wand(operator, mask.as_ref()).await?;
+                            let max_position = postings
+                                .iter()
+                                .map(|posting| posting.term_index() as usize)
+                                .max()
+                                .unwrap_or_default();
+                            let mut tokens_by_position = vec![String::new(); max_position + 1];
+                            for posting in &postings {
+                                let idx = posting.term_index() as usize;
+                                tokens_by_position[idx] = posting.token().to_owned();
+                            }
+                            let use_global_scorer = impact_safe || exact_scoring_required;
+                            let partition_threshold = if use_global_scorer {
+                                impact_shared_threshold
+                            } else {
+                                Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()))
+                            };
+                            let wand_scorer = use_global_scorer.then(|| impact_scorer.clone());
+                            Result::Ok(Some((
+                                part,
+                                docs_for_wand,
+                                postings,
+                                wand_scorer,
+                                partition_threshold,
+                                tokens_by_position,
+                                grouped_expansions,
+                            )))
+                        }
+                    });
+                    let loaded: Vec<_> = stream::iter(loads)
+                        .buffer_unordered(self.store.io_parallelism())
+                        .try_collect::<Vec<_>>()
+                        .await?
+                        .into_iter()
+                        .flatten()
+                        .collect();
+                    if loaded.is_empty() {
+                        return Result::Ok(Vec::new());
                     }
                     let params = params.clone();
                     let mask = mask.clone();
                     let metrics = metrics.clone();
-                    let part_for_wand = part.clone();
-                    let has_grouped_expansions = !grouped_expansions.is_empty();
-                    let wand_params = if has_grouped_expansions {
-                        let mut rescoring_params = params.as_ref().clone();
-                        rescoring_params.limit =
-                            grouped_rescore_wand_limit(params.limit, &grouped_expansions);
-                        Arc::new(rescoring_params)
-                    } else {
-                        params.clone()
-                    };
-                    let partition_threshold = if has_grouped_expansions {
-                        Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()))
-                    } else {
-                        shared_threshold
-                    };
-                    let candidates = spawn_cpu(move || {
-                        let candidates = part_for_wand.bm25_search(
-                            docs_for_wand.as_ref(),
-                            wand_params.as_ref(),
-                            operator,
-                            mask,
+                    let results = spawn_cpu(move || {
+                        let mut results = Vec::with_capacity(loaded.len());
+                        for (
+                            part,
+                            docs_for_wand,
                             postings,
-                            metrics.as_ref(),
+                            wand_scorer,
                             partition_threshold,
-                        )?;
-                        std::result::Result::<_, Error>::Ok(candidates)
+                            tokens_by_position,
+                            grouped_expansions,
+                        ) in loaded
+                        {
+                            let candidates = part.bm25_search(
+                                docs_for_wand.as_ref(),
+                                params.as_ref(),
+                                operator,
+                                mask.clone(),
+                                postings,
+                                wand_scorer,
+                                metrics.as_ref(),
+                                partition_threshold,
+                            )?;
+                            results.push((
+                                part,
+                                PartitionCandidates {
+                                    tokens_by_position,
+                                    grouped_expansions,
+                                    candidates,
+                                },
+                            ));
+                        }
+                        std::result::Result::<_, Error>::Ok(results)
                     })
                     .await?;
-                    let mut partition_result = PartitionCandidates {
-                        tokens_by_position,
-                        grouped_expansions,
-                        candidates,
-                    };
-                    resolve_deferred_candidates(&part.docs, &mut partition_result.candidates)
-                        .await?;
-                    Result::Ok(partition_result)
+                    Result::Ok(results)
                 }
             })
             .collect::<Vec<_>>();
-        let mut parts = stream::iter(parts).buffer_unordered(get_num_compute_intensive_cpus());
+        let mut parts = stream::iter(parts)
+            .buffer_unordered(get_num_compute_intensive_cpus().min(32))
+            .map_ok(|results| stream::iter(results.into_iter().map(Result::Ok)))
+            .try_flatten();
         let mut idf_cache: HashMap<String, f32> = HashMap::new();
-        while let Some(res) = parts.try_next().await? {
+        // Partitions that produced candidates, indexed by the `slot` carried
+        // in deferred heap entries.
+        let mut resolving_parts: Vec<Arc<InvertedPartition>> = Vec::new();
+        while let Some((part, res)) = parts.try_next().await? {
             if res.candidates.is_empty() {
                 continue;
             }
+            let slot = resolving_parts.len() as u32;
+            resolving_parts.push(part);
             let PartitionCandidates {
                 tokens_by_position,
                 grouped_expansions,
@@ -926,7 +1094,7 @@ impl InvertedIndex {
                         score += idf_by_position[term_index as usize]
                             * scorer.doc_weight(freq, doc_length);
                     }
-                    push_scored_candidate(&mut candidates, limit, addr, score)?;
+                    push_scored_candidate(&mut candidates, limit, slot, addr, score);
                 }
             } else {
                 let grouped_positions = grouped_expansions
@@ -950,31 +1118,64 @@ impl InvertedIndex {
                             * scorer.doc_weight(freq, doc_length);
                     }
                     for group in &grouped_expansions {
-                        for term in &group.terms {
+                        for term in group.terms.iter() {
                             let Some(freq) = term.frequency(posting_doc_id) else {
                                 continue;
                             };
-                            let idf_weight = match idf_cache.get(&term.token) {
-                                Some(weight) => *weight,
-                                None => {
-                                    let weight = scorer.query_weight(&term.token);
-                                    idf_cache.insert(term.token.clone(), weight);
-                                    weight
-                                }
-                            };
-                            score += idf_weight * scorer.doc_weight(freq, doc_length);
+                            score += term.query_weight() * scorer.doc_weight(freq, doc_length);
                         }
                     }
-                    push_scored_candidate(&mut candidates, limit, addr, score)?;
+                    push_scored_candidate(&mut candidates, limit, slot, addr, score);
                 }
             }
         }
 
-        Ok(candidates
-            .into_sorted_vec()
-            .into_iter()
-            .map(|Reverse(doc)| (doc.row_id, doc.score.0))
-            .unzip())
+        // Resolve row_ids only for the candidates that survived the global
+        // top-k: group deferred survivors per partition and batch-resolve —
+        // at most `limit` lookups regardless of how many partitions
+        // contributed candidates.
+        /// One partition's surviving deferred candidates: positions in the
+        /// merged result list paired with the doc_ids to resolve.
+        type DeferredGroup = Vec<(usize, u32)>;
+        let sorted = candidates.into_sorted_vec();
+        let mut row_ids = Vec::with_capacity(sorted.len());
+        let mut scores = Vec::with_capacity(sorted.len());
+        let mut deferred: HashMap<u32, DeferredGroup> = HashMap::new();
+        for (pos, Reverse(candidate)) in sorted.into_iter().enumerate() {
+            scores.push(candidate.score.0);
+            match candidate.addr {
+                CandidateAddr::RowId(row_id) => row_ids.push(row_id),
+                CandidateAddr::Pending(doc_id) => {
+                    deferred
+                        .entry(candidate.slot)
+                        .or_default()
+                        .push((pos, doc_id));
+                    // Placeholder, overwritten by the batch resolution below.
+                    row_ids.push(0);
+                }
+            }
+        }
+        if !deferred.is_empty() {
+            let groups = deferred
+                .into_iter()
+                .map(|(slot, entries)| (resolving_parts[slot as usize].clone(), entries))
+                .collect::<Vec<_>>();
+            let resolved: Vec<(DeferredGroup, Vec<u64>)> =
+                stream::iter(groups.into_iter().map(|(part, entries)| async move {
+                    let doc_ids: Vec<u32> = entries.iter().map(|&(_, doc_id)| doc_id).collect();
+                    let resolved = part.docs.resolve_row_ids(&doc_ids).await?;
+                    Result::Ok((entries, resolved))
+                }))
+                .buffer_unordered(get_num_compute_intensive_cpus())
+                .try_collect()
+                .await?;
+            for (entries, resolved) in resolved {
+                for ((pos, _), row_id) in entries.into_iter().zip(resolved) {
+                    row_ids[pos] = row_id;
+                }
+            }
+        }
+        Ok((row_ids, scores))
     }
 
     async fn load_legacy_index(
@@ -1046,6 +1247,37 @@ impl InvertedIndex {
         self.partitions.len() == 1 && self.partitions[0].is_legacy()
     }
 
+    /// Read only the index's [`InvertedIndexParams`],
+    /// Contains more complete info than manifest's lossy `InvertedIndexDetails`.
+    pub async fn load_params(store: &dyn IndexStore) -> Result<InvertedIndexParams> {
+        match store.open_index_file(METADATA_FILE).await {
+            Ok(reader) => {
+                let params = reader
+                    .schema()
+                    .metadata
+                    .get("params")
+                    .ok_or(Error::index("params not found in metadata".to_owned()))?;
+                Ok(serde_json::from_str::<InvertedIndexParams>(params)?)
+            }
+            Err(metadata_error) => {
+                // Legacy format: params live in the tokens file (see
+                // `load_legacy_index`). Some S3 configurations return 403 for
+                // a missing object, so the readable legacy file is the
+                // authoritative format probe.
+                let Ok(reader) = store.open_index_file(TOKENS_FILE).await else {
+                    return Err(metadata_error);
+                };
+                Ok(reader
+                    .schema()
+                    .metadata
+                    .get("tokenizer")
+                    .map(|s| serde_json::from_str::<InvertedIndexParams>(s))
+                    .transpose()?
+                    .unwrap_or_default())
+            }
+        }
+    }
+
     pub async fn load(
         store: Arc<dyn IndexStore>,
         frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
@@ -1192,6 +1424,89 @@ const PREWARM_MAX_CHUNK_TOKENS: usize = 256 * 1024;
 /// Floor on token rows per chunk, so a partition always makes progress.
 const PREWARM_MIN_CHUNK_TOKENS: usize = 1;
 
+/// Maximum number of posting lists in a runtime synthetic cache group. This is
+/// deliberately token-count based so grouping works for old v2 indexes without
+/// scanning posting lengths or requiring index rebuilds.
+static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock<usize> = LazyLock::new(|| {
+    std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
+        .unwrap_or_else(|_| "128".to_string())
+        .parse()
+        .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
+});
+
+fn runtime_posting_group_tokens() -> usize {
+    (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1)
+}
+
+/// Runtime posting-list cache grouping. Non-empty v2 indexes synthesize fixed
+/// groups at read time so prewarm and queries share group cache entries without
+/// persisted grouping metadata or index rebuilds.
+#[derive(Debug, Clone, DeepSizeOf)]
+enum PostingGrouping {
+    /// Leaves legacy or empty partitions ungrouped.
+    None,
+    /// Uses a fixed runtime cache group size measured in token rows, not posting bytes.
+    SyntheticFixed { group_size: u32 },
+}
+
+impl PostingGrouping {
+    fn for_reader(is_legacy_layout: bool, token_count: usize) -> Self {
+        if is_legacy_layout || token_count == 0 {
+            return Self::None;
+        }
+
+        let group_size = u32::try_from(runtime_posting_group_tokens())
+            .unwrap_or(u32::MAX)
+            .max(1);
+        Self::SyntheticFixed { group_size }
+    }
+
+    fn is_grouped(&self) -> bool {
+        !matches!(self, Self::None)
+    }
+
+    fn range_for_token(&self, token_id: u32, token_count: usize) -> Option<(u32, u32)> {
+        match self {
+            Self::None => None,
+            Self::SyntheticFixed { group_size } => {
+                let token_count = u32::try_from(token_count).unwrap_or(u32::MAX);
+                let start = (token_id / *group_size) * *group_size;
+                let end = start.saturating_add(*group_size).min(token_count);
+                Some((start, end))
+            }
+        }
+    }
+
+    fn aligned_chunk_end(&self, token_count: usize, tok_start: usize, desired_end: usize) -> usize {
+        match self {
+            Self::None => desired_end,
+            Self::SyntheticFixed { group_size } => synthetic_group_aligned_chunk_end(
+                usize::try_from(*group_size).unwrap_or(usize::MAX).max(1),
+                token_count,
+                tok_start,
+                desired_end,
+            ),
+        }
+    }
+
+    fn ranges_for_chunk(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        token_count: usize,
+    ) -> Vec<(u32, u32)> {
+        match self {
+            Self::None => Vec::new(),
+            Self::SyntheticFixed { group_size } => synthetic_group_ranges_for_chunk(
+                usize::try_from(*group_size).unwrap_or(usize::MAX).max(1),
+                tok_start,
+                tok_end,
+                token_count,
+            ),
+        }
+    }
+}
+
 /// Token rows per chunk: byte target / average bytes-per-token, clamped to `[MIN, MAX]`.
 fn prewarm_chunk_tokens(token_count: usize, file_size_bytes: u64) -> usize {
     if token_count == 0 {
@@ -1202,11 +1517,8 @@ fn prewarm_chunk_tokens(token_count: usize, file_size_bytes: u64) -> usize {
     by_bytes.clamp(PREWARM_MIN_CHUNK_TOKENS, PREWARM_MAX_CHUNK_TOKENS)
 }
 
-/// Snap a chunk's exclusive token end back to a posting-group boundary so no group
-/// straddles chunks. Returns the largest group boundary in `(tok_start, desired_end]`,
-/// or the next boundary past an oversized group so it runs as one solo chunk.
-fn group_aligned_chunk_end(
-    starts: &[u32],
+fn synthetic_group_aligned_chunk_end(
+    group_size: usize,
     token_count: usize,
     tok_start: usize,
     desired_end: usize,
@@ -1215,21 +1527,38 @@ fn group_aligned_chunk_end(
         return token_count;
     }
 
-    let first_after_start = starts.partition_point(|&start| start as usize <= tok_start);
-    let first_after_desired = starts.partition_point(|&start| start as usize <= desired_end);
-    if first_after_desired > first_after_start {
-        return starts[first_after_desired - 1] as usize;
+    let boundary = desired_end - (desired_end % group_size);
+    if boundary > tok_start {
+        boundary
+    } else {
+        tok_start.saturating_add(group_size).min(token_count)
     }
+}
 
-    // Oversized group: extend to its end so it runs as one chunk.
-    starts
-        .get(first_after_start)
-        .map(|&start| start as usize)
-        .unwrap_or(token_count)
+fn synthetic_group_ranges_for_chunk(
+    group_size: usize,
+    tok_start: usize,
+    tok_end: usize,
+    token_count: usize,
+) -> Vec<(u32, u32)> {
+    let mut ranges = Vec::new();
+    let mut start = tok_start - (tok_start % group_size);
+    if start < tok_start {
+        start = start.saturating_add(group_size).min(token_count);
+    }
+    while start < tok_end {
+        let end = start.saturating_add(group_size).min(token_count);
+        ranges.push((
+            u32::try_from(start).unwrap_or(u32::MAX),
+            u32::try_from(end).unwrap_or(u32::MAX),
+        ));
+        start = end;
+    }
+    ranges
 }
 
 fn prewarm_chunk_ranges(
-    group_starts: Option<&[u32]>,
+    grouping: &PostingGrouping,
     token_count: usize,
     chunk_tokens: usize,
 ) -> Vec<(usize, usize)> {
@@ -1238,8 +1567,8 @@ fn prewarm_chunk_ranges(
     while tok_start < token_count {
         let mut tok_end = (tok_start + chunk_tokens).min(token_count);
         // `tok_start` is always a group boundary; snap `tok_end` back to one too.
-        if let Some(starts) = group_starts {
-            tok_end = group_aligned_chunk_end(starts, token_count, tok_start, tok_end);
+        if grouping.is_grouped() {
+            tok_end = grouping.aligned_chunk_end(token_count, tok_start, tok_end);
         }
         ranges.push((tok_start, tok_end));
         tok_start = tok_end;
@@ -1247,35 +1576,69 @@ fn prewarm_chunk_ranges(
     ranges
 }
 
-fn group_start_indices_for_chunk(starts: &[u32], tok_start: usize, tok_end: usize) -> Range<usize> {
-    let first = starts.partition_point(|&start| (start as usize) < tok_start);
-    let end = starts.partition_point(|&start| (start as usize) < tok_end);
-    first..end
-}
-
-fn group_range_for_start_index(starts: &[u32], token_count: usize, group_idx: usize) -> (u32, u32) {
-    let start = starts[group_idx];
-    let end = starts
-        .get(group_idx + 1)
-        .copied()
-        .unwrap_or(token_count as u32);
-    (start, end)
-}
-
 impl InvertedIndex {
     pub async fn prewarm_with_options(&self, options: &FtsPrewarmOptions) -> Result<()> {
         let with_position = options.with_position;
         let chunk_concurrency = self.store.io_parallelism().max(1);
+        let prewarm_started = Instant::now();
+        info!(
+            partition_count = self.partitions.len(),
+            with_position, chunk_concurrency, "fts index prewarm started"
+        );
         for part in &self.partitions {
-            part.inverted_list
+            let partition_started = Instant::now();
+            info!(
+                partition_id = part.id(),
+                token_count = part.tokens.len(),
+                with_position,
+                chunk_concurrency,
+                "fts partition prewarm started"
+            );
+            if let Err(err) = part
+                .inverted_list
                 .prewarm_posting_lists(with_position, chunk_concurrency)
-                .await?;
+                .await
+            {
+                warn!(
+                    partition_id = part.id(),
+                    error = %err,
+                    elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                    "fts partition posting list prewarm failed"
+                );
+                return Err(err);
+            }
+            info!(
+                partition_id = part.id(),
+                elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                "fts partition posting lists prewarmed"
+            );
             // Materialize the deferred DocSet too: prewarm's contract is
             // that subsequent queries do no IO, so the per-doc row_ids /
             // num_tokens must be resident, not lazily faulted in at query
             // time. `ensure_loaded` opens, reads, and drops the reader.
-            part.docs.ensure_loaded().await?;
+            let docs_started = Instant::now();
+            if let Err(err) = part.docs.ensure_loaded().await {
+                warn!(
+                    partition_id = part.id(),
+                    error = %err,
+                    elapsed_ms = docs_started.elapsed().as_millis() as u64,
+                    total_elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                    "fts partition docset prewarm failed"
+                );
+                return Err(err);
+            }
+            info!(
+                partition_id = part.id(),
+                docset_elapsed_ms = docs_started.elapsed().as_millis() as u64,
+                elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                "fts partition prewarm finished"
+            );
         }
+        info!(
+            partition_count = self.partitions.len(),
+            elapsed_ms = prewarm_started.elapsed().as_millis() as u64,
+            "fts index prewarm finished"
+        );
         Ok(())
     }
     /// Search docs match the input text.
@@ -1460,9 +1823,13 @@ impl InvertedPartition {
         let docs = Arc::new(LazyDocSet::new(
             store.clone(),
             docs_path,
+            id,
+            WeakLanceCache::from(index_cache),
             num_docs,
             false,
             frag_reuse_index,
+            // 256-document blocks score with quantized document lengths.
+            inverted_list.block_size() == MAX_POSTING_BLOCK_SIZE,
         ));
 
         Ok(Self {
@@ -1484,47 +1851,29 @@ impl InvertedPartition {
         let mut new_positions = Vec::with_capacity(new_tokens.capacity());
         let mut seen = HashSet::new();
         for token_idx in 0..tokens.len() {
-            if new_tokens.len() >= params.max_expansions {
+            let remaining = params.max_expansions.saturating_sub(new_tokens.len());
+            if remaining == 0 {
                 break;
             }
             let token = tokens.get_token(token_idx);
             let position = tokens.position(token_idx);
-            let fuzziness = match params.fuzziness {
-                Some(fuzziness) => fuzziness,
-                None => MatchQuery::auto_fuzziness(token),
-            };
-            let lev = fst::automaton::Levenshtein::new(token, fuzziness)
-                .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?;
-
-            let base_len = tokens.token_type().prefix_len(token) as u32;
-            if let TokenMap::Fst(ref map) = self.tokens.tokens {
-                let mut expanded = Vec::new();
-                let remaining = params.max_expansions - new_tokens.len();
-                match base_len + params.prefix_length {
-                    0 => take_fst_keys(map.search(lev), &mut expanded, remaining),
-                    prefix_length => {
-                        let prefix = &token[..min(prefix_length as usize, token.len())];
-                        let prefix = fst::automaton::Str::new(prefix).starts_with();
-                        take_fst_keys(
-                            map.search(lev.intersection(prefix)),
-                            &mut expanded,
-                            remaining,
-                        )
-                    }
+            let base_prefix_len = tokens.token_type().prefix_len(token) as u32;
+            let mut candidates = BTreeSet::new();
+            self.collect_fuzzy_candidates(
+                token,
+                base_prefix_len,
+                params,
+                remaining,
+                &mut candidates,
+            )?;
+            for candidate in candidates {
+                if new_tokens.len() >= params.max_expansions {
+                    break;
                 }
-                for token in expanded {
-                    if seen.insert((token.clone(), position)) {
-                        new_tokens.push(token);
-                        new_positions.push(position);
-                        if new_tokens.len() >= params.max_expansions {
-                            break;
-                        }
-                    }
+                if seen.insert((candidate.clone(), position)) {
+                    new_tokens.push(candidate);
+                    new_positions.push(position);
                 }
-            } else {
-                return Err(Error::index(
-                    "tokens is not fst, which is not expected".to_owned(),
-                ));
             }
         }
         Ok(Tokens::with_positions(
@@ -1534,34 +1883,193 @@ impl InvertedPartition {
         ))
     }
 
-    fn union_plain_posting_lists(postings: Vec<PostingList>) -> Result<PostingList> {
-        let mut freqs_by_row_id = BTreeMap::new();
-        for posting in postings {
-            for (row_id, freq, _) in posting.iter() {
-                let entry = freqs_by_row_id.entry(row_id).or_insert(0u32);
-                *entry = entry.checked_add(freq).ok_or_else(|| {
-                    Error::index(format!("posting frequency overflow for row id {}", row_id))
-                })?;
+    /// Collect up to `limit` fuzzy candidates for one query token from this
+    /// partition's token FST, in key (lexicographic) order. Callers merge
+    /// candidates across partitions and apply the query-wide
+    /// `max_expansions` budget; truncating each partition at `limit` is
+    /// lossless for that selection because any term among the merged
+    /// lexicographically-smallest `limit` is also among its own partition's
+    /// smallest `limit`.
+    fn collect_fuzzy_candidates(
+        &self,
+        token: &str,
+        base_prefix_len: u32,
+        params: &FtsSearchParams,
+        limit: usize,
+        candidates: &mut BTreeSet<String>,
+    ) -> Result<()> {
+        let fuzziness = match params.fuzziness {
+            Some(fuzziness) => fuzziness,
+            None => MatchQuery::auto_fuzziness(token),
+        };
+        let lev = fst::automaton::Levenshtein::new(token, fuzziness)
+            .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?;
+
+        if let TokenMap::Fst(ref map) = self.tokens.tokens {
+            let mut expanded = Vec::new();
+            match base_prefix_len + params.prefix_length {
+                0 => take_fst_keys(map.search(lev), &mut expanded, limit),
+                prefix_length => {
+                    let prefix = &token[..min(prefix_length as usize, token.len())];
+                    let prefix = fst::automaton::Str::new(prefix).starts_with();
+                    take_fst_keys(map.search(lev.intersection(prefix)), &mut expanded, limit)
+                }
             }
+            candidates.extend(expanded);
+            Ok(())
+        } else {
+            Err(Error::index(
+                "tokens is not fst, which is not expected".to_owned(),
+            ))
         }
-        let mut row_ids = Vec::with_capacity(freqs_by_row_id.len());
-        let mut frequencies = Vec::with_capacity(freqs_by_row_id.len());
-        for (row_id, freq) in freqs_by_row_id {
-            row_ids.push(row_id);
-            frequencies.push(freq as f32);
-        }
-        Ok(PostingList::Plain(PlainPostingList::new(
-            ScalarBuffer::from(row_ids),
-            ScalarBuffer::from(frequencies),
-            None,
-            None,
+    }
+
+    #[inline]
+    fn grouped_score_upper_bound(
+        query_weight: f32,
+        union_freq: u32,
+        doc_length: u32,
+        scorer: &MemBM25Scorer,
+    ) -> f32 {
+        // BM25's document weight is monotonic in frequency and every IDF is
+        // non-negative. Scoring the summed frequency with the summed IDF is
+        // therefore an upper bound on the sum of the individual term scores.
+        query_weight * scorer.doc_weight(union_freq, doc_length)
+    }
+
+    fn grouped_block_max_scores(
+        doc_ids: &[u32],
+        frequencies: &[u32],
+        block_size: usize,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Vec<f32> {
+        doc_ids
+            .chunks(block_size)
+            .zip(frequencies.chunks(block_size))
+            .map(|(doc_ids, frequencies)| {
+                doc_ids
+                    .iter()
+                    .zip(frequencies)
+                    .map(|(doc_id, freq)| {
+                        Self::grouped_score_upper_bound(
+                            query_weight,
+                            *freq,
+                            docs.scoring_num_tokens(*doc_id),
+                            scorer,
+                        )
+                    })
+                    .fold(0.0, f32::max)
+            })
+            .collect()
+    }
+
+    fn union_plain_posting_lists(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
+        let mut freqs_by_row_id = BTreeMap::new();
+        for posting in postings {
+            for (row_id, freq, _) in posting.iter() {
+                let entry = freqs_by_row_id.entry(row_id).or_insert(0u32);
+                *entry = entry.checked_add(freq).ok_or_else(|| {
+                    Error::index(format!("posting frequency overflow for row id {}", row_id))
+                })?;
+            }
+        }
+        let mut row_ids = Vec::with_capacity(freqs_by_row_id.len());
+        let mut frequencies = Vec::with_capacity(freqs_by_row_id.len());
+        let mut max_score = 0.0_f32;
+        for (row_id, freq) in freqs_by_row_id {
+            max_score = max_score.max(Self::grouped_score_upper_bound(
+                query_weight,
+                freq,
+                docs.num_tokens_by_row_id(row_id),
+                scorer,
+            ));
+            row_ids.push(row_id);
+            frequencies.push(freq as f32);
+        }
+        Ok(PostingList::Plain(PlainPostingList::new(
+            ScalarBuffer::from(row_ids),
+            ScalarBuffer::from(frequencies),
+            Some(max_score),
+            None,
+        )))
+    }
+
+    fn union_plain_posting_lists_with_positions(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
+        let mut positions_by_row_id = BTreeMap::<u64, Vec<u32>>::new();
+        for posting in postings {
+            for (row_id, _, positions) in posting.iter() {
+                let positions = positions.ok_or_else(|| {
+                    Error::index("cannot union grouped phrase terms without positions".to_string())
+                })?;
+                positions_by_row_id
+                    .entry(row_id)
+                    .or_default()
+                    .extend(positions);
+            }
+        }
+        if positions_by_row_id.is_empty() {
+            return Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            )));
+        }
+
+        let mut row_ids = Vec::with_capacity(positions_by_row_id.len());
+        let mut frequencies = Vec::with_capacity(positions_by_row_id.len());
+        let mut positions_builder = ListBuilder::new(Int32Builder::new());
+        let mut max_score = 0.0_f32;
+        for (row_id, mut positions) in positions_by_row_id {
+            positions.sort_unstable();
+            let frequency = positions.len() as u32;
+            max_score = max_score.max(Self::grouped_score_upper_bound(
+                query_weight,
+                frequency,
+                docs.num_tokens_by_row_id(row_id),
+                scorer,
+            ));
+            row_ids.push(row_id);
+            frequencies.push(frequency as f32);
+            for position in positions {
+                positions_builder.values().append_value(position as i32);
+            }
+            positions_builder.append(true);
+        }
+
+        Ok(PostingList::Plain(PlainPostingList::new(
+            ScalarBuffer::from(row_ids),
+            ScalarBuffer::from(frequencies),
+            Some(max_score),
+            Some(positions_builder.finish()),
         )))
     }
 
     fn union_compressed_posting_lists(
         postings: Vec<PostingList>,
         docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
     ) -> Result<PostingList> {
+        let block_size = postings
+            .iter()
+            .find_map(|posting| match posting {
+                PostingList::Compressed(posting) => Some(posting.block_size),
+                PostingList::Plain(_) => None,
+            })
+            .unwrap_or(LEGACY_BLOCK_SIZE);
         let mut freqs_by_doc_id = BTreeMap::new();
         for posting in postings {
             for (doc_id, freq, _) in posting.iter() {
@@ -1586,7 +2094,7 @@ impl InvertedPartition {
             )));
         }
 
-        let mut builder = PostingListBuilder::new(false);
+        let mut builder = PostingListBuilder::new_with_block_size(false, block_size);
         let mut doc_ids = Vec::with_capacity(freqs_by_doc_id.len());
         let mut frequencies = Vec::with_capacity(freqs_by_doc_id.len());
         for (doc_id, freq) in freqs_by_doc_id {
@@ -1594,14 +2102,91 @@ impl InvertedPartition {
             doc_ids.push(doc_id);
             frequencies.push(freq);
         }
-        let block_max_scores = docs.calculate_block_max_scores(doc_ids.iter(), frequencies.iter());
+        let block_max_scores = Self::grouped_block_max_scores(
+            &doc_ids,
+            &frequencies,
+            block_size,
+            docs,
+            query_weight,
+            scorer,
+        );
+        let batch = builder.to_batch(block_max_scores)?;
+        let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+        let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+        PostingList::from_batch(&batch, Some(max_score), Some(length))
+    }
+
+    fn union_compressed_posting_lists_with_positions(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
+        let block_size = postings
+            .iter()
+            .find_map(|posting| match posting {
+                PostingList::Compressed(posting) => Some(posting.block_size),
+                PostingList::Plain(_) => None,
+            })
+            .unwrap_or(LEGACY_BLOCK_SIZE);
+        let mut positions_by_doc_id = BTreeMap::<u32, Vec<u32>>::new();
+        for posting in postings {
+            for (doc_id, _, positions) in posting.iter() {
+                let doc_id = u32::try_from(doc_id).map_err(|_| {
+                    Error::index(format!(
+                        "compressed posting doc id {} exceeds u32::MAX",
+                        doc_id
+                    ))
+                })?;
+                let positions = positions.ok_or_else(|| {
+                    Error::index("cannot union grouped phrase terms without positions".to_string())
+                })?;
+                positions_by_doc_id
+                    .entry(doc_id)
+                    .or_default()
+                    .extend(positions);
+            }
+        }
+        if positions_by_doc_id.is_empty() {
+            return Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            )));
+        }
+
+        let mut builder = PostingListBuilder::new_with_block_size(true, block_size);
+        let mut doc_ids = Vec::with_capacity(positions_by_doc_id.len());
+        let mut frequencies = Vec::with_capacity(positions_by_doc_id.len());
+        for (doc_id, mut positions) in positions_by_doc_id {
+            positions.sort_unstable();
+            let frequency = positions.len() as u32;
+            builder.add(doc_id, PositionRecorder::Position(positions.into()));
+            doc_ids.push(doc_id);
+            frequencies.push(frequency);
+        }
+        let block_max_scores = Self::grouped_block_max_scores(
+            &doc_ids,
+            &frequencies,
+            block_size,
+            docs,
+            query_weight,
+            scorer,
+        );
         let batch = builder.to_batch(block_max_scores)?;
         let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
         let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
         PostingList::from_batch(&batch, Some(max_score), Some(length))
     }
 
-    fn union_posting_lists(postings: Vec<PostingList>, docs: &DocSet) -> Result<PostingList> {
+    fn union_posting_lists(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        with_positions: bool,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
         let has_plain = postings
             .iter()
             .any(|posting| matches!(posting, PostingList::Plain(_)));
@@ -1612,8 +2197,19 @@ impl InvertedPartition {
             (true, true) => Err(Error::index(
                 "cannot union mixed plain and compressed posting lists".to_owned(),
             )),
-            (true, false) => Self::union_plain_posting_lists(postings),
-            (false, true) => Self::union_compressed_posting_lists(postings, docs),
+            (true, false) if with_positions => {
+                Self::union_plain_posting_lists_with_positions(postings, docs, query_weight, scorer)
+            }
+            (true, false) => Self::union_plain_posting_lists(postings, docs, query_weight, scorer),
+            (false, true) if with_positions => Self::union_compressed_posting_lists_with_positions(
+                postings,
+                docs,
+                query_weight,
+                scorer,
+            ),
+            (false, true) => {
+                Self::union_compressed_posting_lists(postings, docs, query_weight, scorer)
+            }
             (false, false) => Ok(PostingList::Plain(PlainPostingList::new(
                 ScalarBuffer::from(Vec::<u64>::new()),
                 ScalarBuffer::from(Vec::<f32>::new()),
@@ -1632,9 +2228,9 @@ impl InvertedPartition {
         tokens: &Tokens,
         params: &FtsSearchParams,
         operator: Operator,
+        impact_scorer: &MemBM25Scorer,
         metrics: &dyn MetricsCollector,
     ) -> Result<LoadedPostings> {
-        let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0);
         let is_phrase_query = params.phrase_slop.is_some();
         let is_and_query = operator == Operator::And;
         let required_positions = (is_and_query || is_phrase_query).then(|| {
@@ -1642,13 +2238,18 @@ impl InvertedPartition {
                 .map(|index| tokens.position(index))
                 .collect::<HashSet<_>>()
         });
-        let tokens = match is_fuzzy {
-            true => self.expand_fuzzy(tokens, params)?,
-            false => tokens.clone(),
-        };
+        // Fuzzy expansion already ran once at the index level (see
+        // `InvertedIndex::bm25_search`) under the global `max_expansions`
+        // budget. Positions identify alternatives that must share one posting
+        // iterator, including code identifier subwords and fuzzy expansions.
+        let tokens = tokens.clone();
         let token_positions = (0..tokens.len())
             .map(|index| tokens.position(index))
             .collect::<Vec<_>>();
+        let mut seen_positions = HashSet::with_capacity(token_positions.len());
+        let exact_scoring_required = token_positions
+            .iter()
+            .any(|position| !seen_positions.insert(*position));
         let mut token_ids = Vec::with_capacity(tokens.len());
         let mut matched_positions = required_positions.as_ref().map(|_| HashSet::new());
         for (index, token) in tokens.into_iter().enumerate() {
@@ -1659,9 +2260,6 @@ impl InvertedPartition {
                     matched_positions.insert(position);
                 }
                 token_ids.push((token_id, token, position));
-            } else if is_phrase_query || is_and_query {
-                // if the token is not found, we can't do phrase or AND query
-                return Ok(LoadedPostings::empty());
             }
         }
         if token_ids.is_empty() {
@@ -1674,16 +2272,8 @@ impl InvertedPartition {
             return Ok(LoadedPostings::empty());
         }
 
-        let is_fuzzy_and_query = is_fuzzy && is_and_query && !is_phrase_query;
-        if !is_phrase_query {
-            if is_fuzzy_and_query {
-                token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id));
-                token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2);
-            } else {
-                token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id);
-                token_ids.dedup_by_key(|(token_id, _, _)| *token_id);
-            }
-        }
+        token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id));
+        token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2);
 
         let num_docs = self.docs.len();
         let loaded_postings = stream::iter(token_ids)
@@ -1699,8 +2289,11 @@ impl InvertedPartition {
             .try_collect::<Vec<_>>()
             .await?;
 
+        let needs_union = loaded_postings
+            .windows(2)
+            .any(|window| window[0].2 == window[1].2);
         if (is_and_query || is_phrase_query)
-            && !is_fuzzy_and_query
+            && !needs_union
             && loaded_postings
                 .iter()
                 .any(|(_, _, _, posting)| posting.is_empty())
@@ -1708,29 +2301,42 @@ impl InvertedPartition {
             return Ok(LoadedPostings::empty());
         }
 
-        if !is_fuzzy_and_query {
+        if !needs_union {
+            let impact_safe = loaded_postings
+                .iter()
+                .all(|(_, _, _, posting)| posting.has_impacts());
             return Ok(LoadedPostings {
                 postings: loaded_postings
                     .into_iter()
                     .map(|(token_id, token, position, posting)| {
-                        let query_weight = idf(posting.len(), num_docs);
-                        PostingIterator::with_query_weight(
+                        let needs_scorer_upper_bound =
+                            exact_scoring_required && !posting.has_impacts();
+                        let query_weight = if impact_safe || exact_scoring_required {
+                            impact_scorer.query_weight(&token)
+                        } else {
+                            idf(posting.len(), num_docs)
+                        };
+                        let posting = PostingIterator::with_query_weight(
                             token,
                             token_id,
                             position,
                             query_weight,
                             posting,
                             num_docs,
-                        )
+                        );
+                        if needs_scorer_upper_bound {
+                            posting.with_scorer_upper_bound()
+                        } else {
+                            posting
+                        }
                     })
                     .collect(),
                 grouped_expansions: Vec::new(),
+                impact_safe,
+                exact_scoring_required,
             });
         }
 
-        let needs_union = loaded_postings
-            .windows(2)
-            .any(|window| window[0].2 == window[1].2);
         let docs_for_union = if needs_union {
             Some(self.docs.ensure_num_tokens_loaded().await?)
         } else {
@@ -1755,43 +2361,77 @@ impl InvertedPartition {
             } else {
                 let token_id = group[0].0;
                 let token = group[0].1.clone();
+                let terms = group
+                    .iter()
+                    .map(|(_, token, posting)| {
+                        GroupedTermScorer::new(impact_scorer.query_weight(token), posting)
+                    })
+                    .collect::<Vec<_>>();
+                let terms = Arc::<[GroupedTermScorer]>::from(terms);
+                let query_weight = terms.iter().map(GroupedTermScorer::query_weight).sum();
                 grouped_expansions.push(GroupedExpansionTerms {
                     position,
-                    terms: group
-                        .iter()
-                        .map(|(_, token, posting)| ExpansionTermFreqs::new(token.clone(), posting))
-                        .collect(),
+                    terms: terms.clone(),
                 });
                 let postings = group
                     .into_iter()
                     .map(|(_, _, posting)| posting)
                     .collect::<Vec<_>>();
+                let docs = docs_for_union.as_deref().ok_or_else(|| {
+                    Error::index("union docs were not loaded for grouped query terms".to_string())
+                })?;
                 let posting = Self::union_posting_lists(
                     postings,
-                    docs_for_union
-                        .as_deref()
-                        .expect("union docs must be loaded for grouped fuzzy AND"),
+                    docs,
+                    is_phrase_query,
+                    query_weight,
+                    impact_scorer,
                 )?;
-                (token_id, token, posting)
+                if posting.is_empty() && (is_and_query || is_phrase_query) {
+                    return Ok(LoadedPostings::empty());
+                }
+                grouped_postings.push(
+                    PostingIterator::with_query_weight(
+                        token,
+                        token_id,
+                        position,
+                        query_weight,
+                        posting,
+                        num_docs,
+                    )
+                    .with_grouped_terms(terms),
+                );
+                continue;
             };
             if posting.is_empty() {
-                return Ok(LoadedPostings::empty());
+                if is_and_query || is_phrase_query {
+                    return Ok(LoadedPostings::empty());
+                }
+                continue;
             }
 
-            let query_weight = idf(posting.len(), num_docs);
-            grouped_postings.push(PostingIterator::with_query_weight(
+            let query_weight = impact_scorer.query_weight(&token);
+            let needs_scorer_upper_bound = !posting.has_impacts();
+            let posting = PostingIterator::with_query_weight(
                 token,
                 token_id,
                 position,
                 query_weight,
                 posting,
                 num_docs,
-            ));
+            );
+            grouped_postings.push(if needs_scorer_upper_bound {
+                posting.with_scorer_upper_bound()
+            } else {
+                posting
+            });
         }
 
         Ok(LoadedPostings {
             postings: grouped_postings,
             grouped_expansions,
+            impact_safe: false,
+            exact_scoring_required: true,
         })
     }
 
@@ -1807,6 +2447,7 @@ impl InvertedPartition {
         operator: Operator,
         mask: Arc<RowAddrMask>,
         postings: Vec<PostingIterator>,
+        impact_scorer: Option<Arc<MemBM25Scorer>>,
         metrics: &dyn MetricsCollector,
         shared_threshold: Arc<AtomicU32>,
     ) -> Result<Vec<DocCandidate>> {
@@ -1817,25 +2458,29 @@ impl InvertedPartition {
         // Caller selects the DocSet shape via `LazyDocSet::docs_for_wand`
         // and passes it in here; wand uses `docs.has_row_ids()` to
         // handle the num_tokens-only case.
-        let scorer = IndexBM25Scorer::new(std::iter::once(self));
-        let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer)
-            .with_shared_threshold(shared_threshold);
-        let hits = wand.search(params, mask, metrics)?;
+        let hits = if let Some(scorer) = impact_scorer {
+            let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer)
+                .with_shared_threshold(shared_threshold);
+            wand.search(params, mask, metrics)?
+        } else {
+            let scorer = IndexBM25Scorer::new(std::iter::once(self));
+            let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer)
+                .with_shared_threshold(shared_threshold);
+            wand.search(params, mask, metrics)?
+        };
         Ok(hits)
     }
 
     pub async fn into_builder(self) -> Result<InnerBuilder> {
-        let mut builder = InnerBuilder::new_with_posting_tail_codec(
+        let mut builder = InnerBuilder::new_with_posting_tail_codec_and_block_size(
             self.id,
             self.inverted_list.has_positions(),
             self.token_set_format,
             self.inverted_list.posting_tail_codec(),
+            self.inverted_list.block_size(),
         );
         builder.tokens = self.tokens.into_mutable();
-        // into_builder rewrites every doc, so materialize the full
-        // DocSet now and clone it out of the Arc.
-        let docs_arc = self.docs.ensure_loaded().await?;
-        builder.docs = (*docs_arc).clone();
+        builder.docs = self.docs.owned_docset().await?;
 
         builder
             .posting_lists
@@ -2238,14 +2883,15 @@ pub struct PostingListReader {
     metadata: PostingMetadata,
 
     has_position: bool,
+    has_impacts: bool,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     positions_layout: PositionsLayout,
 
-    /// First row of each posting-list cache group, decoded at open from the
-    /// global buffer named by [`POSTING_GROUP_OFFSETS_BUF_KEY`] (issue #7040).
-    /// `None` for indexes written before grouping; those use the per-token
-    /// cache path. Always present for grouped v2 indexes with `>0` rows.
-    group_starts: Option<Vec<u32>>,
+    /// Runtime posting-list cache grouping. Non-empty v2 indexes use synthetic
+    /// fixed groups so prewarm can improve cache density without rebuilding the
+    /// index or relying on persisted grouping metadata.
+    grouping: PostingGrouping,
 
     index_cache: WeakLanceCache,
 }
@@ -2320,7 +2966,7 @@ impl DeepSizeOf for PostingListReader {
                 })
                 .unwrap_or(0),
         };
-        metadata_size + self.group_starts.deep_size_of_children(context)
+        metadata_size + self.grouping.deep_size_of_children(context)
     }
 }
 
@@ -2337,7 +2983,9 @@ impl PostingListReader {
             PositionsLayout::None
         };
         let posting_tail_codec = parse_posting_tail_codec(&reader.schema().metadata)?;
+        let block_size = parse_posting_block_size(&reader.schema().metadata)?;
         let has_position = positions_layout != PositionsLayout::None;
+        let has_impacts = reader.schema().field(IMPACT_COL).is_some();
         let metadata = if reader.schema().field(POSTING_COL).is_none() {
             let (offsets, max_scores) = Self::load_metadata(reader.schema())?;
             PostingMetadata::LegacyV1 {
@@ -2350,36 +2998,22 @@ impl PostingListReader {
             }
         };
 
-        let group_starts = Self::load_group_starts(reader.as_ref()).await?;
+        let is_legacy_layout = matches!(&metadata, PostingMetadata::LegacyV1 { .. });
+        let grouping = PostingGrouping::for_reader(is_legacy_layout, reader.num_rows());
 
         Ok(Self {
             reader,
             metadata,
             has_position,
+            has_impacts,
             posting_tail_codec,
+            block_size,
             positions_layout,
-            group_starts,
+            grouping,
             index_cache: WeakLanceCache::from(index_cache),
         })
     }
 
-    /// Decode the posting-list cache-group boundaries from the global buffer
-    /// recorded in schema metadata, if present (issue #7040). Returns `None`
-    /// for indexes written before grouping was introduced.
-    async fn load_group_starts(reader: &dyn IndexReader) -> Result<Option<Vec<u32>>> {
-        let Some(buf_id) = reader.schema().metadata.get(POSTING_GROUP_OFFSETS_BUF_KEY) else {
-            return Ok(None);
-        };
-        let buf_id: u32 = buf_id.parse().map_err(|e| {
-            Error::index(format!(
-                "invalid {POSTING_GROUP_OFFSETS_BUF_KEY} metadata value {buf_id:?}: {e}"
-            ))
-        })?;
-        let bytes = reader.read_global_buffer(buf_id).await?;
-        let group_starts = decode_group_starts(&bytes)?;
-        Ok(Some(group_starts))
-    }
-
     // for legacy format
     // returns the offsets and max scores
     fn load_metadata(
@@ -2419,6 +3053,10 @@ impl PostingListReader {
         self.posting_tail_codec
     }
 
+    pub(crate) fn block_size(&self) -> usize {
+        self.block_size
+    }
+
     fn is_legacy_layout(&self) -> bool {
         matches!(self.metadata, PostingMetadata::LegacyV1 { .. })
     }
@@ -2547,7 +3185,7 @@ impl PostingListReader {
             self.posting_batch_legacy(token_id, with_position).await
         } else {
             let token_id = token_id as usize;
-            let columns = if with_position {
+            let mut columns = if with_position {
                 match self.positions_layout {
                     PositionsLayout::SharedStream(_) => {
                         vec![
@@ -2562,6 +3200,9 @@ impl PostingListReader {
             } else {
                 vec![POSTING_COL]
             };
+            if self.has_impacts {
+                columns.push(IMPACT_COL);
+            }
             let batch = self
                 .reader
                 .read_range(token_id..token_id + 1, Some(&columns))
@@ -2606,39 +3247,49 @@ impl PostingListReader {
             Some((start, end)) => {
                 let group = self
                     .index_cache
-                    .get_or_insert_with_key(PostingListGroupKey { start, end }, || async move {
-                        metrics.record_part_load();
-                        info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start);
-                        self.load_posting_list_group(start, end).await
-                    })
+                    .get_or_insert_with_key(
+                        posting_list_group_cache_key(start, end, self.has_impacts),
+                        || async move {
+                            metrics.record_part_load();
+                            info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start);
+                            self.load_posting_list_group(start, end).await
+                        },
+                    )
                     .await?;
+                let (max_score, length) = if group.needs_external_metadata() {
+                    self.posting_metadata_for_token(token_id).await?
+                } else {
+                    (None, None)
+                };
                 let slot = (token_id - start) as usize;
                 group
-                    .get(slot)
+                    .posting_list(slot, max_score, length)?
                     .ok_or_else(|| {
                         Error::index(format!(
                             "token {token_id} maps to slot {slot} outside posting group [{start}, {end})"
                         ))
                     })?
-                    .clone()
             }
-            // Fallback for indexes written before grouping: one cache entry
-            // per token.
+            // Fallback for layouts that cannot use row-based groups: one cache
+            // entry per token.
             None => self
                 .index_cache
-                .get_or_insert_with_key(PostingListKey { token_id }, || async move {
-                    metrics.record_part_load();
-                    info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id);
-                    // Fetch the posting batch and this token's (max_score,
-                    // length) in parallel; for cold v2 partitions this is one
-                    // single-row metadata read plus one posting-row read,
-                    // instead of pulling the full per-token metadata table.
-                    let (batch, (max_score, length)) = futures::try_join!(
-                        self.posting_batch(token_id, false),
-                        self.posting_metadata_for_token(token_id),
-                    )?;
-                    self.posting_list_from_batch(&batch, max_score, length)
-                })
+                .get_or_insert_with_key(
+                    posting_list_cache_key(token_id, self.has_impacts),
+                    || async move {
+                        metrics.record_part_load();
+                        info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id);
+                        // Fetch the posting batch and this token's (max_score,
+                        // length) in parallel; for cold v2 partitions this is one
+                        // single-row metadata read plus one posting-row read,
+                        // instead of pulling the full per-token metadata table.
+                        let (batch, (max_score, length)) = futures::try_join!(
+                            self.posting_batch(token_id, false),
+                            self.posting_metadata_for_token(token_id),
+                        )?;
+                        self.posting_list_from_batch(&batch, max_score, length)
+                    },
+                )
                 .await?
                 .as_ref()
                 .clone(),
@@ -2654,55 +3305,30 @@ impl PostingListReader {
     }
 
     /// Map a token id to its cache group's row range `[start, end)`, or `None`
-    /// when grouping is not available (pre-grouping indexes) so the caller
-    /// falls back to the per-token path. In v2 the token id is the row offset,
-    /// so the group range is also the physical row range.
+    /// when grouping is not available so the caller falls back to the per-token
+    /// path. In v2 the token id is the row offset, so the group range is also
+    /// the physical row range.
     fn group_range_for_token(&self, token_id: u32) -> Option<(u32, u32)> {
-        let starts = self.group_starts.as_ref()?;
-        // partition_point returns the count of group starts <= token_id, so the
-        // owning group begins at index k - 1 and the next start (if any) is its
-        // exclusive end.
-        let k = starts.partition_point(|&s| s <= token_id);
-        // k == 0 means token_id precedes the first group start, which cannot
-        // happen for a valid token in a grouped index (the first group starts
-        // at row 0); guard anyway and fall back to the per-token path.
-        if k == 0 {
-            return None;
-        }
-        let start = starts[k - 1];
-        // The last group runs to the final posting list. `self.len()` is the
-        // authoritative posting-list count (offsets length for v1, row count for
-        // v2), and prewarm derives the same `end` from it — so warm- and
-        // cold-cache group keys are identical by construction, not by the
-        // incidental v2 `num_rows == token_count` equality.
-        let end = starts.get(k).copied().unwrap_or(self.len() as u32);
-        Some((start, end))
+        self.grouping.range_for_token(token_id, self.len())
     }
 
-    /// Read rows `[start, end)` of the posting file and decode them into a
-    /// [`PostingListGroup`] cache value (issue #7040). Positions are excluded;
-    /// phrase queries load them on demand via [`Self::read_positions`].
+    /// Read rows `[start, end)` into one compact Arrow-backed cache value.
+    /// Positions are excluded; phrase queries load them on demand via
+    /// [`Self::read_positions`].
     async fn load_posting_list_group(&self, start: u32, end: u32) -> Result<PostingListGroup> {
+        let mut columns = vec![POSTING_COL, MAX_SCORE_COL, LENGTH_COL];
+        if self.has_impacts {
+            columns.push(IMPACT_COL);
+        }
         let batch = self
             .reader
-            .read_range(
-                start as usize..end as usize,
-                Some(&[POSTING_COL, MAX_SCORE_COL, LENGTH_COL]),
-            )
+            .read_range(start as usize..end as usize, Some(&columns))
             .await?;
-        let max_scores = batch[MAX_SCORE_COL].as_primitive::<Float32Type>();
-        let lengths = batch[LENGTH_COL].as_primitive::<UInt32Type>();
-        let mut posting_lists = Vec::with_capacity(batch.num_rows());
-        for i in 0..batch.num_rows() {
-            let row = batch.slice(i, 1);
-            let posting = self.posting_list_from_batch(
-                &row,
-                Some(max_scores.value(i)),
-                Some(lengths.value(i)),
-            )?;
-            posting_lists.push(posting);
-        }
-        Ok(PostingListGroup::new(posting_lists))
+        PostingListGroup::new_packed_with_block_size(
+            batch.shrink_to_fit()?,
+            self.posting_tail_codec,
+            self.block_size,
+        )
     }
 
     fn posting_list_from_batch_parts(
@@ -2710,6 +3336,7 @@ impl PostingListReader {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions_layout: PositionsLayout,
     ) -> Result<PostingList> {
         let posting_list = PostingList::from_batch_with_tail_codec_and_positions_layout(
@@ -2717,6 +3344,7 @@ impl PostingListReader {
             max_score,
             length,
             posting_tail_codec,
+            block_size,
             positions_layout,
         )?;
         Ok(posting_list)
@@ -2733,6 +3361,7 @@ impl PostingListReader {
             max_score,
             length,
             self.posting_tail_codec,
+            self.block_size,
             self.positions_layout,
         )
     }
@@ -2769,6 +3398,7 @@ impl PostingListReader {
                 ctx.max_scores.map(|scores| scores[global]),
                 ctx.lengths.map(|lengths| lengths[global]),
                 ctx.posting_tail_codec,
+                ctx.block_size,
                 ctx.positions_layout,
             )?;
             posting_lists.push((global as u32, posting_list));
@@ -2826,43 +3456,63 @@ impl PostingListReader {
             ));
         }
 
-        // Make sure max_scores/lengths are populated before we clone them into
-        // the blocking task; otherwise the v2 branch would unwrap empty
-        // OnceCells.
+        // Make max_scores/lengths available for query-local packed views. The
+        // materialized fallback also clones them into its blocking build task.
         self.ensure_metadata_loaded().await?;
 
-        let state = self.chunk_build_state();
-        // With grouping the cache stores one entry per group, so a group's posting
-        // lists must all be resident at once: align chunk boundaries to whole
-        // groups. Without grouping, chunks are plain token ranges.
-        let group_starts = self.group_starts.clone();
+        // With grouping the cache stores one entry per group, so a group's
+        // posting lists must all be resident at once: align chunk boundaries to
+        // whole groups. Without grouping, chunks are plain token ranges.
+        let grouping = self.grouping.clone();
+        let use_packed_groups = grouping.is_grouped() && !with_position;
+        // Packed groups reuse the reader's bulk metadata at query time, so they
+        // do not need the temporary full-partition metadata clones used by the
+        // materialized fallback.
+        let state = (!use_packed_groups).then(|| self.chunk_build_state());
         let token_count = self.len();
         let posting_data_size_bytes = self.posting_data_size_bytes();
         let chunk_tokens = chunk_tokens_override
             .unwrap_or_else(|| prewarm_chunk_tokens(token_count, posting_data_size_bytes))
             .max(1);
-        let chunk_ranges = prewarm_chunk_ranges(group_starts.as_deref(), token_count, chunk_tokens);
+        let chunk_ranges = prewarm_chunk_ranges(&grouping, token_count, chunk_tokens);
         let chunk_count = chunk_ranges.len();
         let chunk_concurrency = chunk_concurrency.max(1);
 
         let read_build_start = Instant::now();
         stream::iter(chunk_ranges)
             .map(|(tok_start, tok_end)| {
-                let state = &state;
-                let group_starts = group_starts.as_deref();
+                let state = state.as_ref();
+                let grouping = &grouping;
                 async move {
-                    let posting_lists = self
-                        .build_chunk_postings(tok_start, tok_end, with_position, state)
-                        .await?;
-                    self.publish_chunk_postings(
-                        posting_lists,
-                        group_starts,
-                        tok_start,
-                        tok_end,
-                        token_count,
-                        with_position,
-                    )
-                    .await;
+                    if use_packed_groups {
+                        let groups = self
+                            .build_packed_chunk_groups(tok_start, tok_end, token_count, grouping)
+                            .await?;
+                        for (start, end, group) in groups {
+                            self.index_cache
+                                .insert_with_key(
+                                    &posting_list_group_cache_key(start, end, self.has_impacts),
+                                    Arc::new(group),
+                                )
+                                .await;
+                        }
+                    } else {
+                        let state = state.expect(
+                            "materialized prewarm must initialize posting-list build state",
+                        );
+                        let posting_lists = self
+                            .build_chunk_postings(tok_start, tok_end, with_position, state)
+                            .await?;
+                        self.publish_chunk_postings(
+                            posting_lists,
+                            grouping,
+                            tok_start,
+                            tok_end,
+                            token_count,
+                            with_position,
+                        )
+                        .await;
+                    }
                     Result::Ok(())
                 }
             })
@@ -2905,6 +3555,7 @@ impl PostingListReader {
             max_scores: max_scores.map(Arc::new),
             lengths: lengths.map(Arc::new),
             posting_tail_codec: self.posting_tail_codec,
+            block_size: self.block_size,
             positions_layout: self.positions_layout,
         }
     }
@@ -2938,12 +3589,14 @@ impl PostingListReader {
         let max_scores = state.max_scores.clone();
         let lengths = state.lengths.clone();
         let posting_tail_codec = state.posting_tail_codec;
+        let block_size = state.block_size;
         let positions_layout = state.positions_layout;
         let posting_lists = spawn_blocking(move || {
             let ctx = PrewarmBuildCtx {
                 max_scores: max_scores.as_deref().map(|v| v.as_slice()),
                 lengths: lengths.as_deref().map(|v| v.as_slice()),
                 posting_tail_codec,
+                block_size,
                 positions_layout,
             };
             let chunk = PrewarmChunk {
@@ -2972,6 +3625,52 @@ impl PostingListReader {
         Ok(posting_lists)
     }
 
+    /// Build compact v2 groups directly from one posting-row chunk. Each group
+    /// slice is deep-copied once, so it owns only its Arrow buffers without
+    /// materializing a `Vec<PostingList>` or retaining the full chunk.
+    async fn build_packed_chunk_groups(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        token_count: usize,
+        grouping: &PostingGrouping,
+    ) -> Result<Vec<(u32, u32, PostingListGroup)>> {
+        debug_assert!(grouping.is_grouped());
+        debug_assert!(!self.is_legacy_layout());
+
+        let chunk_batch = self.read_chunk_batch(tok_start, tok_end, false).await?;
+        let ranges = grouping.ranges_for_chunk(tok_start, tok_end, token_count);
+        let posting_tail_codec = self.posting_tail_codec;
+        let block_size = self.block_size;
+
+        spawn_blocking(move || {
+            let mut groups = Vec::with_capacity(ranges.len());
+            for (start, end) in ranges {
+                let start_usize = start as usize;
+                let end_usize = end as usize;
+                let local_start = start_usize - tok_start;
+                let group_len = end_usize - start_usize;
+                let group_batch = chunk_batch.slice(local_start, group_len).shrink_to_fit()?;
+                groups.push((
+                    start,
+                    end,
+                    PostingListGroup::new_packed_with_block_size(
+                        group_batch,
+                        posting_tail_codec,
+                        block_size,
+                    )?,
+                ));
+            }
+            Result::Ok(groups)
+        })
+        .await
+        .map_err(|err| {
+            Error::internal(format!(
+                "Failed to build packed prewarm posting groups in blocking task: {err}"
+            ))
+        })?
+    }
+
     /// Strip positions into their own per-token cache entries (the posting cache
     /// holds positions-free lists), then populate the same cache keys the read
     /// path uses: grouped entries when grouping is active, per-token entries
@@ -2979,14 +3678,26 @@ impl PostingListReader {
     async fn publish_chunk_postings(
         &self,
         posting_lists: Vec<(u32, PostingList)>,
-        group_starts: Option<&[u32]>,
+        grouping: &PostingGrouping,
         tok_start: usize,
         tok_end: usize,
         token_count: usize,
         with_position: bool,
     ) {
-        match group_starts {
-            Some(starts) => {
+        match grouping {
+            PostingGrouping::None => {
+                for (token_id, mut posting_list) in posting_lists {
+                    self.cache_positions(&mut posting_list, token_id, with_position)
+                        .await;
+                    self.index_cache
+                        .insert_with_key(
+                            &posting_list_cache_key(token_id, self.has_impacts),
+                            Arc::new(posting_list),
+                        )
+                        .await;
+                }
+            }
+            PostingGrouping::SyntheticFixed { .. } => {
                 let mut chunk_postings = Vec::with_capacity(posting_lists.len());
                 for (token_id, mut posting_list) in posting_lists {
                     self.cache_positions(&mut posting_list, token_id, with_position)
@@ -2997,23 +3708,16 @@ impl PostingListReader {
                 // in it; `chunk_postings[i]` is token `tok_start + i`. The last
                 // group's `end` derives from `token_count`, matching the read path
                 // so both produce identical `PostingListGroupKey`s.
-                for group_idx in group_start_indices_for_chunk(starts, tok_start, tok_end) {
-                    let (start, end) = group_range_for_start_index(starts, token_count, group_idx);
+                for (start, end) in grouping.ranges_for_chunk(tok_start, tok_end, token_count) {
                     let start_usize = start as usize;
                     let lo = start_usize - tok_start;
                     let hi = end as usize - tok_start;
                     let group = PostingListGroup::new(chunk_postings[lo..hi].to_vec());
                     self.index_cache
-                        .insert_with_key(&PostingListGroupKey { start, end }, Arc::new(group))
-                        .await;
-                }
-            }
-            None => {
-                for (token_id, mut posting_list) in posting_lists {
-                    self.cache_positions(&mut posting_list, token_id, with_position)
-                        .await;
-                    self.index_cache
-                        .insert_with_key(&PostingListKey { token_id }, Arc::new(posting_list))
+                        .insert_with_key(
+                            &posting_list_group_cache_key(start, end, self.has_impacts),
+                            Arc::new(group),
+                        )
                         .await;
                 }
             }
@@ -3183,6 +3887,9 @@ impl PostingListReader {
                 }
             }
         }
+        if self.has_impacts {
+            base_columns.push(IMPACT_COL);
+        }
         base_columns
     }
 }
@@ -3194,6 +3901,7 @@ struct ChunkBuildState {
     max_scores: Option<Arc<Vec<f32>>>,
     lengths: Option<Arc<Vec<u32>>>,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     positions_layout: PositionsLayout,
 }
 
@@ -3204,6 +3912,7 @@ struct PrewarmBuildCtx<'a> {
     max_scores: Option<&'a [f32]>,
     lengths: Option<&'a [u32]>,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     positions_layout: PositionsLayout,
 }
 
@@ -3297,8 +4006,8 @@ impl CacheKey for PostingListKey {
 
 /// Cache key for a group of consecutive posting lists stored as a single
 /// entry, covering rows `[start, end)` (issue #7040). The range, not a token
-/// id, is the key so that a write-time config change that reshapes groups
-/// simply misses old entries instead of serving a differently-shaped group.
+/// id, is the key so a runtime group-size change simply misses old entries
+/// instead of serving a differently-shaped group.
 #[derive(Debug, Clone)]
 pub struct PostingListGroupKey {
     pub start: u32,
@@ -3321,6 +4030,52 @@ impl CacheKey for PostingListGroupKey {
     }
 }
 
+/// Internal cache-key decorator that isolates impact-bearing posting values
+/// without changing the source-compatible public posting key structs.
+#[derive(Debug, Clone)]
+struct ImpactAwareCacheKey<K> {
+    inner: K,
+    has_impacts: bool,
+}
+
+impl<K: CacheKey> CacheKey for ImpactAwareCacheKey<K> {
+    type ValueType = K::ValueType;
+
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        if self.has_impacts {
+            format!("{}-impacts", self.inner.key()).into()
+        } else {
+            self.inner.key()
+        }
+    }
+
+    fn type_name() -> &'static str {
+        K::type_name()
+    }
+
+    fn codec() -> Option<CacheCodec> {
+        K::codec()
+    }
+}
+
+fn posting_list_cache_key(token_id: u32, has_impacts: bool) -> ImpactAwareCacheKey<PostingListKey> {
+    ImpactAwareCacheKey {
+        inner: PostingListKey { token_id },
+        has_impacts,
+    }
+}
+
+fn posting_list_group_cache_key(
+    start: u32,
+    end: u32,
+    has_impacts: bool,
+) -> ImpactAwareCacheKey<PostingListGroupKey> {
+    ImpactAwareCacheKey {
+        inner: PostingListGroupKey { start, end },
+        has_impacts,
+    }
+}
+
 #[derive(Debug, Clone, DeepSizeOf)]
 struct PostingMetadataValue {
     max_score: f32,
@@ -3435,27 +4190,352 @@ impl SharedPositionStream {
 }
 
 /// A group of consecutive posting lists held in a single cache entry, in row
-/// order (issue #7040). `posting_lists[i]` corresponds to row `start + i`,
-/// where `start` is the group's first row from [`PostingListGroupKey`].
-#[derive(Debug, Clone, DeepSizeOf)]
+/// order (issue #7040). Prewarmed modern groups without positions retain only
+/// the compact Arrow posting rows read from `invert.lance`; max-score/length
+/// metadata stays in the reader and is injected when a query creates a
+/// posting-list view. Cold-loaded groups may keep inline metadata to preserve
+/// one-read query loading. Legacy and position-bearing prewarm paths use the
+/// materialized fallback.
+#[derive(Debug, Clone)]
 pub struct PostingListGroup {
-    pub(super) posting_lists: Vec<PostingList>,
+    pub(super) storage: PostingListGroupStorage,
+}
+
+#[derive(Debug, Clone)]
+pub(super) enum PostingListGroupStorage {
+    Packed(PackedPostingListGroup),
+    Materialized(Vec<PostingList>),
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct PackedPostingListGroup {
+    pub(super) batch: RecordBatch,
+    pub(super) posting_tail_codec: PostingTailCodec,
+    pub(super) block_size: usize,
+    first_docs_states: Arc<[OnceLock<Box<[u32]>>]>,
+    first_docs_state_capacity_bytes: usize,
+    impact_states: Option<Arc<[OnceLock<Box<ImpactSkipData>>]>>,
+    impact_state_capacity_bytes: usize,
+}
+
+impl DeepSizeOf for PostingListGroup {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => group
+                .batch
+                .columns()
+                .iter()
+                .map(|column| sliced_cache_bytes(column.as_ref()))
+                .sum::<usize>()
+                .saturating_add(group.first_docs_state_capacity_bytes)
+                .saturating_add(group.impact_state_capacity_bytes),
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                posting_lists.deep_size_of_children(context)
+            }
+        }
+    }
 }
 
 impl PostingListGroup {
     pub(super) fn new(posting_lists: Vec<PostingList>) -> Self {
-        Self { posting_lists }
+        Self {
+            storage: PostingListGroupStorage::Materialized(posting_lists),
+        }
     }
 
-    /// Borrow the posting list at offset `slot` within the group (i.e.
-    /// `token_id - start`).
-    pub(super) fn get(&self, slot: usize) -> Option<&PostingList> {
-        self.posting_lists.get(slot)
+    pub(super) fn new_packed(
+        batch: RecordBatch,
+        posting_tail_codec: PostingTailCodec,
+    ) -> Result<Self> {
+        let block_size = parse_posting_block_size(batch.schema_ref().metadata())?;
+        Self::new_packed_with_block_size(batch, posting_tail_codec, block_size)
     }
-}
 
-#[derive(Debug, Clone, DeepSizeOf)]
-pub enum PostingList {
+    fn new_packed_with_block_size(
+        batch: RecordBatch,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Result<Self> {
+        validate_block_size(block_size)?;
+        if let Some(encoded_block_size) = batch.schema_ref().metadata().get(POSTING_BLOCK_SIZE_KEY)
+        {
+            let encoded_block_size = encoded_block_size.parse::<usize>().map_err(|err| {
+                Error::index(format!(
+                    "invalid {POSTING_BLOCK_SIZE_KEY} metadata value {encoded_block_size:?}: {err}"
+                ))
+            })?;
+            if encoded_block_size != block_size {
+                return Err(Error::index(format!(
+                    "packed posting group {POSTING_BLOCK_SIZE_KEY}={encoded_block_size} does not match block_size={block_size}"
+                )));
+            }
+        }
+
+        // Projected reads may drop schema metadata. Restore the reader's
+        // validated block size before the batch enters the packed cache so IPC
+        // roundtrips remain self-describing. Older packed cache entries omit
+        // the key and enter through new_packed with the legacy 128-doc default.
+        let mut schema = batch.schema().as_ref().clone();
+        schema
+            .metadata
+            .insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string());
+        let batch = batch.with_schema(Arc::new(schema))?;
+        let postings = batch
+            .column_by_name(POSTING_COL)
+            .and_then(|column| column.as_list_opt::<i32>())
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "packed posting group column {POSTING_COL} must be List<LargeBinary>"
+                ))
+            })?;
+        if postings.values().data_type() != &DataType::LargeBinary {
+            return Err(Error::index(format!(
+                "packed posting group column {POSTING_COL} must contain LargeBinary values, got {}",
+                postings.values().data_type()
+            )));
+        }
+        if postings.null_count() != 0 {
+            return Err(Error::index(
+                "packed posting group column must not contain nulls".to_string(),
+            ));
+        }
+        let total_posting_blocks = (0..batch.num_rows())
+            .map(|slot| postings.value_length(slot) as usize)
+            .sum::<usize>();
+        let first_docs_states: Arc<[OnceLock<Box<[u32]>>]> = (0..batch.num_rows())
+            .map(|_| OnceLock::new())
+            .collect::<Vec<_>>()
+            .into();
+        // Reserve the compact per-slot state slab and the block-head arrays it
+        // can lazily retain, so warming these derived values cannot grow the
+        // cache beyond its admission charge.
+        let first_docs_state_capacity_bytes = first_docs_states
+            .len()
+            .saturating_mul(std::mem::size_of::<OnceLock<Box<[u32]>>>())
+            .saturating_add(total_posting_blocks.saturating_mul(std::mem::size_of::<u32>()));
+        let (impact_states, impact_state_capacity_bytes) = if let Some(impacts) =
+            batch.column_by_name(IMPACT_COL)
+        {
+            let impacts = impacts.as_list_opt::<i32>().ok_or_else(|| {
+                Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must be List<LargeBinary>"
+                ))
+            })?;
+            if impacts.values().data_type() != &DataType::LargeBinary {
+                return Err(Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must contain LargeBinary values, got {}",
+                    impacts.values().data_type()
+                )));
+            }
+            if impacts.null_count() != 0 {
+                return Err(Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must not contain nulls"
+                )));
+            }
+            let mut derived_cache_bytes = 0usize;
+            for slot in 0..batch.num_rows() {
+                let posting_blocks = postings.value_length(slot) as usize;
+                let impact_entries = impacts.value_length(slot) as usize;
+                let expected_impact_entries =
+                    posting_blocks.saturating_add(posting_blocks.div_ceil(IMPACT_LEVEL1_BLOCKS));
+                if impact_entries != expected_impact_entries {
+                    return Err(Error::index(format!(
+                        "packed posting group impact slot {slot} has {impact_entries} entries, expected {expected_impact_entries} for {posting_blocks} posting blocks"
+                    )));
+                }
+                derived_cache_bytes = derived_cache_bytes.saturating_add(
+                    ImpactSkipData::derived_cache_bytes_for_entries(impact_entries),
+                );
+            }
+
+            let states: Arc<[OnceLock<Box<ImpactSkipData>>]> = (0..batch.num_rows())
+                .map(|_| OnceLock::new())
+                .collect::<Vec<_>>()
+                .into();
+            // Account up front for every allocation that the lazy states can
+            // eventually retain. The impact entry bytes themselves remain in
+            // `batch` and are already charged exactly once above.
+            let per_slot_bytes = std::mem::size_of::<OnceLock<Box<ImpactSkipData>>>()
+                .saturating_add(std::mem::size_of::<ImpactSkipData>());
+            let capacity_bytes = states
+                .len()
+                .saturating_mul(per_slot_bytes)
+                .saturating_add(derived_cache_bytes);
+            (Some(states), capacity_bytes)
+        } else {
+            (None, 0)
+        };
+        match (
+            batch.column_by_name(MAX_SCORE_COL),
+            batch.column_by_name(LENGTH_COL),
+        ) {
+            (None, None) => {}
+            (Some(max_scores), Some(lengths)) => {
+                let max_scores = max_scores
+                    .as_primitive_opt::<Float32Type>()
+                    .ok_or_else(|| {
+                        Error::index(format!(
+                            "packed posting group column {MAX_SCORE_COL} must be Float32"
+                        ))
+                    })?;
+                let lengths = lengths.as_primitive_opt::<UInt32Type>().ok_or_else(|| {
+                    Error::index(format!(
+                        "packed posting group column {LENGTH_COL} must be UInt32"
+                    ))
+                })?;
+                if max_scores.null_count() != 0 || lengths.null_count() != 0 {
+                    return Err(Error::index(
+                        "packed posting group metadata columns must not contain nulls".to_string(),
+                    ));
+                }
+            }
+            _ => {
+                return Err(Error::index(format!(
+                    "packed posting group must contain both {MAX_SCORE_COL} and {LENGTH_COL}, or neither"
+                )));
+            }
+        }
+
+        Ok(Self {
+            storage: PostingListGroupStorage::Packed(PackedPostingListGroup {
+                batch,
+                posting_tail_codec,
+                block_size,
+                first_docs_states,
+                first_docs_state_capacity_bytes,
+                impact_states,
+                impact_state_capacity_bytes,
+            }),
+        })
+    }
+
+    pub(super) fn len(&self) -> usize {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => group.batch.num_rows(),
+            PostingListGroupStorage::Materialized(posting_lists) => posting_lists.len(),
+        }
+    }
+
+    #[cfg(test)]
+    pub(super) fn is_packed(&self) -> bool {
+        matches!(&self.storage, PostingListGroupStorage::Packed(_))
+    }
+
+    fn needs_external_metadata(&self) -> bool {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => {
+                group.batch.column_by_name(MAX_SCORE_COL).is_none()
+            }
+            PostingListGroupStorage::Materialized(_) => false,
+        }
+    }
+
+    /// Build an owned posting-list view for `slot`. Packed groups clone only
+    /// Arrow array metadata; the compressed posting bytes remain shared with
+    /// the group's `List<LargeBinary>` child buffers.
+    pub(super) fn posting_list(
+        &self,
+        slot: usize,
+        max_score: Option<f32>,
+        length: Option<u32>,
+    ) -> Result<Option<PostingList>> {
+        match &self.storage {
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                Ok(posting_lists.get(slot).cloned())
+            }
+            PostingListGroupStorage::Packed(group) => {
+                if slot >= group.batch.num_rows() {
+                    return Ok(None);
+                }
+                let postings = group
+                    .batch
+                    .column_by_name(POSTING_COL)
+                    .and_then(|column| column.as_list_opt::<i32>())
+                    .ok_or_else(|| {
+                        Error::index(format!(
+                            "packed posting group column {POSTING_COL} must be List<LargeBinary>"
+                        ))
+                    })?;
+                let blocks = postings.value(slot);
+                let blocks = blocks.as_binary_opt::<i64>().ok_or_else(|| {
+                    Error::index(format!(
+                        "packed posting group slot {slot} is not LargeBinary"
+                    ))
+                })?;
+                let max_score = match group.batch.column_by_name(MAX_SCORE_COL) {
+                    Some(column) => column
+                        .as_primitive_opt::<Float32Type>()
+                        .expect("packed group metadata was validated at construction")
+                        .value(slot),
+                    None => max_score.ok_or_else(|| {
+                        Error::index("packed posting group requires max-score metadata".to_string())
+                    })?,
+                };
+                let length = match group.batch.column_by_name(LENGTH_COL) {
+                    Some(column) => column
+                        .as_primitive_opt::<UInt32Type>()
+                        .expect("packed group metadata was validated at construction")
+                        .value(slot),
+                    None => length.ok_or_else(|| {
+                        Error::index("packed posting group requires length metadata".to_string())
+                    })?,
+                };
+                let impacts = match (
+                    group.impact_states.as_ref(),
+                    group.batch.column_by_name(IMPACT_COL),
+                ) {
+                    (Some(states), Some(column)) => {
+                        let state = states.get(slot).ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group impact state missing slot {slot}"
+                            ))
+                        })?;
+                        let impact_lists = column.as_list_opt::<i32>().ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group column {IMPACT_COL} must be List<LargeBinary>"
+                            ))
+                        })?;
+                        let entries = impact_lists.value(slot);
+                        let entries = entries.as_binary_opt::<i64>().ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group impact slot {slot} is not LargeBinary"
+                            ))
+                        })?;
+                        let impacts =
+                            state.get_or_init(|| {
+                                Box::new(ImpactSkipData::new(entries.clone(), blocks.len()).expect(
+                                    "packed impact entry count was validated at construction",
+                                ))
+                            });
+                        Some(impacts.as_ref().clone())
+                    }
+                    (None, None) => None,
+                    _ => {
+                        return Err(Error::internal(
+                            "packed posting group impact column/state mismatch".to_string(),
+                        ));
+                    }
+                };
+                Ok(Some(PostingList::Compressed(
+                    CompressedPostingList::new(
+                        blocks.clone(),
+                        max_score,
+                        length,
+                        group.posting_tail_codec,
+                        group.block_size,
+                        None,
+                        impacts,
+                    )
+                    .with_packed_first_docs(group.first_docs_states.clone(), slot),
+                )))
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, DeepSizeOf)]
+#[allow(clippy::large_enum_variant)]
+pub enum PostingList {
     Plain(PlainPostingList),
     Compressed(CompressedPostingList),
 }
@@ -3467,7 +4547,8 @@ impl PostingList {
         length: Option<u32>,
     ) -> Result<Self> {
         let posting_tail_codec = parse_posting_tail_codec(batch.schema_ref().metadata())?;
-        Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec)
+        let block_size = parse_posting_block_size(batch.schema_ref().metadata())?;
+        Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec, block_size)
     }
 
     pub fn from_batch_with_tail_codec(
@@ -3475,6 +4556,7 @@ impl PostingList {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
     ) -> Result<Self> {
         let positions_layout = if batch.column_by_name(COMPRESSED_POSITION_COL).is_some() {
             PositionsLayout::SharedStream(parse_shared_position_codec(
@@ -3490,6 +4572,7 @@ impl PostingList {
             max_score,
             length,
             posting_tail_codec,
+            block_size,
             positions_layout,
         )
     }
@@ -3499,6 +4582,7 @@ impl PostingList {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions_layout: PositionsLayout,
     ) -> Result<Self> {
         match batch.column_by_name(POSTING_COL) {
@@ -3513,8 +4597,9 @@ impl PostingList {
                     max_score.unwrap(),
                     length.unwrap(),
                     posting_tail_codec,
+                    block_size,
                     shared_position_codec,
-                );
+                )?;
                 Ok(Self::Compressed(posting))
             }
             None => {
@@ -3535,6 +4620,13 @@ impl PostingList {
         }
     }
 
+    pub fn has_impacts(&self) -> bool {
+        match self {
+            Self::Plain(_) => false,
+            Self::Compressed(posting) => posting.impacts.is_some(),
+        }
+    }
+
     pub fn set_positions(&mut self, positions: CompressedPositionStorage) {
         match self {
             Self::Plain(posting) => match positions {
@@ -3584,9 +4676,14 @@ impl PostingList {
             Self::Plain(_) => PostingTailCodec::Fixed32,
             Self::Compressed(posting) => posting.posting_tail_codec,
         };
-        let mut builder = PostingListBuilder::new_with_posting_tail_codec(
+        let block_size = match &self {
+            Self::Plain(_) => LEGACY_BLOCK_SIZE,
+            Self::Compressed(posting) => posting.block_size,
+        };
+        let mut builder = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
             self.has_position(),
             posting_tail_codec,
+            block_size,
         );
         match self {
             // legacy format
@@ -3739,15 +4836,77 @@ impl PlainPostingList {
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, Clone)]
+enum FirstDocsState {
+    Standalone(Arc<OnceLock<Box<[u32]>>>),
+    Packed {
+        states: Arc<[OnceLock<Box<[u32]>>]>,
+        slot: usize,
+    },
+}
+
+impl FirstDocsState {
+    fn standalone() -> Self {
+        Self::Standalone(Arc::new(OnceLock::new()))
+    }
+
+    fn state(&self) -> &OnceLock<Box<[u32]>> {
+        match self {
+            Self::Standalone(state) => state,
+            Self::Packed { states, slot } => &states[*slot],
+        }
+    }
+
+    fn get_or_init(&self, initialize: impl FnOnce() -> Box<[u32]>) -> &[u32] {
+        self.state().get_or_init(initialize)
+    }
+
+    fn capacity_bytes(
+        &self,
+        block_count: usize,
+        context: &mut lance_core::deepsize::Context,
+    ) -> usize {
+        if context.mark_seen(self.state() as *const _ as usize) {
+            std::mem::size_of::<OnceLock<Box<[u32]>>>()
+                .saturating_add(block_count.saturating_mul(std::mem::size_of::<u32>()))
+        } else {
+            0
+        }
+    }
+
+    #[cfg(test)]
+    fn shares_state_with(&self, other: &Self) -> bool {
+        std::ptr::eq(self.state(), other.state())
+    }
+}
+
+#[derive(Debug, Clone)]
 pub struct CompressedPostingList {
     pub max_score: f32,
     pub length: u32,
     // each binary is a block of compressed data
-    // that contains `BLOCK_SIZE` doc ids and then `BLOCK_SIZE` frequencies
+    // that contains `block_size` doc ids and then `block_size` frequencies,
+    // packed by the physical bitpacker matching that block size.
     pub blocks: LargeBinaryArray,
     pub posting_tail_codec: PostingTailCodec,
+    pub block_size: usize,
     pub positions: Option<CompressedPositionStorage>,
+    pub(crate) impacts: Option<ImpactSkipData>,
+    // First doc id per block, baked lazily and shared across per-query clones
+    // of the cached list. See `block_first_docs`.
+    first_docs: FirstDocsState,
+}
+
+impl PartialEq for CompressedPostingList {
+    fn eq(&self, other: &Self) -> bool {
+        self.max_score == other.max_score
+            && self.length == other.length
+            && self.blocks == other.blocks
+            && self.posting_tail_codec == other.posting_tail_codec
+            && self.block_size == other.block_size
+            && self.positions == other.positions
+            && self.impacts == other.impacts
+    }
 }
 
 impl DeepSizeOf for CompressedPostingList {
@@ -3758,33 +4917,68 @@ impl DeepSizeOf for CompressedPostingList {
                 .as_ref()
                 .map(|positions| positions.deep_size_of_children(context))
                 .unwrap_or(0)
+            + self
+                .impacts
+                .as_ref()
+                .map(|impacts| {
+                    sliced_cache_bytes(impacts.entries())
+                        .saturating_add(impacts.derived_cache_bytes())
+                })
+                .unwrap_or(0)
+            + self.first_docs.capacity_bytes(self.blocks.len(), context)
     }
 }
 
 impl CompressedPostingList {
-    pub fn new(
+    pub(crate) fn new(
         blocks: LargeBinaryArray,
         max_score: f32,
         length: u32,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions: Option<CompressedPositionStorage>,
+        impacts: Option<ImpactSkipData>,
     ) -> Self {
+        debug_assert!(block_size.is_power_of_two());
         Self {
             max_score,
             length,
             blocks,
             posting_tail_codec,
+            block_size,
             positions,
+            impacts,
+            first_docs: FirstDocsState::standalone(),
         }
     }
 
+    fn with_packed_first_docs(mut self, states: Arc<[OnceLock<Box<[u32]>>]>, slot: usize) -> Self {
+        debug_assert!(slot < states.len());
+        self.first_docs = FirstDocsState::Packed { states, slot };
+        self
+    }
+
+    /// Block sizes are validated powers of two, so per-doc hot loops derive
+    /// block indices with shift/mask instead of runtime division, which is
+    /// measurably slower in the iterator advance path.
+    #[inline]
+    pub(crate) fn block_shift(&self) -> u32 {
+        self.block_size.trailing_zeros()
+    }
+
+    #[inline]
+    pub(crate) fn block_mask(&self) -> usize {
+        self.block_size - 1
+    }
+
     pub fn from_batch(
         batch: &RecordBatch,
         max_score: f32,
         length: u32,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         shared_position_codec: Option<PositionStreamCodec>,
-    ) -> Self {
+    ) -> Result<Self> {
         debug_assert_eq!(batch.num_rows(), 1);
         let blocks = batch[POSTING_COL]
             .as_list::<i32>()
@@ -3813,14 +5007,24 @@ impl CompressedPostingList {
                 )
             })
         };
+        let impacts = batch
+            .column_by_name(IMPACT_COL)
+            .map(|col| {
+                let entries = col.as_list::<i32>().value(0).as_binary::<i64>().clone();
+                ImpactSkipData::new(entries, blocks.len())
+            })
+            .transpose()?;
 
-        Self {
+        Ok(Self {
             max_score,
             length,
             blocks,
             posting_tail_codec,
+            block_size,
             positions,
-        }
+            impacts,
+            first_docs: FirstDocsState::standalone(),
+        })
     }
 
     pub fn iter(&self) -> CompressedPostingListIterator {
@@ -3829,23 +5033,58 @@ impl CompressedPostingList {
             self.blocks.clone(),
             self.posting_tail_codec,
             self.positions.clone(),
+            self.block_size,
         )
     }
 
     pub fn block_max_score(&self, block_idx: usize) -> f32 {
+        // 256-document blocks store no per-block max score: their impact
+        // skip data supplies the tight per-block bound, so callers on that
+        // path never reach here. Fall back to the list-level max, which is
+        // still a valid (looser) bound for any block.
+        if super::encoding::posting_block_score_prefix_len(self.block_size) == 0 {
+            return self.max_score;
+        }
         let block = self.blocks.value(block_idx);
         block[0..4].try_into().map(f32::from_le_bytes).unwrap()
     }
 
+    #[inline]
     pub fn block_least_doc_id(&self, block_idx: usize) -> u32 {
-        let block = self.blocks.value(block_idx);
-        let remainder = self.length as usize % BLOCK_SIZE;
-        let is_remainder_block = remainder > 0 && block_idx + 1 == self.blocks.len();
-        if is_remainder_block {
-            super::encoding::read_posting_tail_first_doc(block, self.posting_tail_codec)
-        } else {
-            block[4..8].try_into().map(u32::from_le_bytes).unwrap()
-        }
+        self.block_first_docs()[block_idx]
+    }
+
+    /// First doc id of every block, decoded once per cached list and shared by
+    /// the per-query clones. Block boundary lookups (window bounds, block
+    /// binary searches) are hot enough that re-reading the block headers —
+    /// and re-decoding the tail block — shows up in profiles.
+    pub(crate) fn block_first_docs(&self) -> &[u32] {
+        self.first_docs.get_or_init(|| {
+            (0..self.blocks.len())
+                .map(|block_idx| {
+                    let block = self.blocks.value(block_idx);
+                    let remainder = self.length as usize % self.block_size;
+                    if block_idx + 1 == self.blocks.len() && remainder > 0 {
+                        return super::encoding::read_posting_tail_first_doc(
+                            block,
+                            self.posting_tail_codec,
+                            self.block_size,
+                        );
+                    }
+                    let prefix = super::encoding::posting_block_score_prefix_len(self.block_size);
+                    block[prefix..prefix + 4]
+                        .try_into()
+                        .map(u32::from_le_bytes)
+                        .unwrap()
+                })
+                .collect::<Vec<_>>()
+                .into_boxed_slice()
+        })
+    }
+
+    #[cfg(test)]
+    fn shares_first_docs_with(&self, other: &Self) -> bool {
+        self.first_docs.shares_state_with(&other.first_docs)
     }
 }
 
@@ -3896,12 +5135,14 @@ impl EncodedBlocks {
         doc_ids: &[u32],
         frequencies: &[u32],
         codec: PostingTailCodec,
+        block_size: usize,
     ) -> Result<()> {
         self.offsets.push(self.bytes.len() as u32);
         super::encoding::encode_remainder_posting_block_into(
             doc_ids,
             frequencies,
             codec,
+            block_size,
             &mut self.bytes,
         )
     }
@@ -3970,6 +5211,7 @@ pub struct PostingListBuilder {
     open_doc_id: Option<u32>,
     open_doc_frequency: u32,
     open_doc_last_position: Option<u32>,
+    block_size: usize,
     memory_size_bytes: u32,
     len: u32,
 }
@@ -3977,14 +5219,11 @@ pub struct PostingListBuilder {
 pub(super) struct PostingListBatchBuilder {
     schema: SchemaRef,
     postings: ListBuilder<LargeBinaryBuilder>,
+    impacts: Option<ListBuilder<LargeBinaryBuilder>>,
     max_scores: Float32Builder,
     lengths: UInt32Builder,
     positions: BatchPositionsBuilder,
     len: usize,
-    /// Tracks posting-list cache-group boundaries in row order across all
-    /// batches this builder produces (issue #7040). Outlives `finish`, which
-    /// only resets the per-batch column builders.
-    group_accumulator: PostingGroupAccumulator,
 }
 
 enum BatchPositionsBuilder {
@@ -3999,6 +5238,7 @@ enum BatchPositionsBuilder {
 struct PostingListParts<'a> {
     with_positions: bool,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     length: usize,
     encoded_blocks: EncodedBlocks,
     encoded_position_blocks: EncodedPositionBlocks,
@@ -4012,7 +5252,6 @@ impl PostingListBatchBuilder {
         with_positions: bool,
         format_version: InvertedListFormatVersion,
         capacity: usize,
-        group_config: PostingGroupConfig,
     ) -> Self {
         let positions = if !with_positions {
             BatchPositionsBuilder::None
@@ -4027,14 +5266,18 @@ impl PostingListBatchBuilder {
                 capacity,
             ))
         };
+        let impacts = schema
+            .field_with_name(IMPACT_COL)
+            .ok()
+            .map(|_| ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity));
         Self {
             schema,
             postings: ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity),
+            impacts,
             max_scores: Float32Builder::with_capacity(capacity),
             lengths: UInt32Builder::with_capacity(capacity),
             positions,
             len: 0,
-            group_accumulator: PostingGroupAccumulator::new(group_config),
         }
     }
 
@@ -4049,11 +5292,11 @@ impl PostingListBatchBuilder {
     fn append(
         &mut self,
         compressed: LargeBinaryArray,
+        impacts: Option<&ImpactSkipData>,
         max_score: f32,
         length: u32,
         positions: Option<&CompressedPositionStorage>,
     ) -> Result<()> {
-        let posting_bytes = compressed.value_data().len();
         {
             let values = self.postings.values();
             for index in 0..compressed.len() {
@@ -4061,7 +5304,19 @@ impl PostingListBatchBuilder {
             }
         }
         self.postings.append(true);
-        self.group_accumulator.push(posting_bytes);
+        if let Some(impacts_builder) = &mut self.impacts {
+            let impacts = impacts.ok_or_else(|| {
+                Error::index(format!(
+                    "impacts builder missing impact data for posting length {}",
+                    length
+                ))
+            })?;
+            let values = impacts_builder.values();
+            for index in 0..impacts.entries().len() {
+                values.append_value(impacts.entries().value(index));
+            }
+            impacts_builder.append(true);
+        }
         self.max_scores.append_value(max_score);
         self.lengths.append_value(length);
 
@@ -4126,6 +5381,9 @@ impl PostingListBatchBuilder {
             Arc::new(self.max_scores.finish()) as ArrayRef,
             Arc::new(self.lengths.finish()) as ArrayRef,
         ];
+        if let Some(impacts) = &mut self.impacts {
+            columns.push(Arc::new(impacts.finish()) as ArrayRef);
+        }
         match &mut self.positions {
             BatchPositionsBuilder::None => {}
             BatchPositionsBuilder::Legacy(position_lists) => {
@@ -4142,13 +5400,6 @@ impl PostingListBatchBuilder {
         self.len = 0;
         RecordBatch::try_new(self.schema.clone(), columns).map_err(Error::from)
     }
-
-    /// Consume the builder and return the posting-list cache-group boundaries
-    /// accumulated across all batches (issue #7040). Each entry is the first
-    /// row of a group; the sequence is monotonically increasing.
-    pub fn into_group_starts(self) -> Vec<u32> {
-        self.group_accumulator.into_starts()
-    }
 }
 
 impl PostingListBuilder {
@@ -4161,9 +5412,10 @@ impl PostingListBuilder {
     }
 
     pub fn new(with_position: bool) -> Self {
-        Self::new_with_posting_tail_codec(
+        Self::new_with_posting_tail_codec_and_block_size(
             with_position,
             current_fts_format_version().posting_tail_codec(),
+            LEGACY_BLOCK_SIZE,
         )
     }
 
@@ -4171,6 +5423,27 @@ impl PostingListBuilder {
         with_position: bool,
         posting_tail_codec: PostingTailCodec,
     ) -> Self {
+        Self::new_with_posting_tail_codec_and_block_size(
+            with_position,
+            posting_tail_codec,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_block_size(with_position: bool, block_size: usize) -> Self {
+        Self::new_with_posting_tail_codec_and_block_size(
+            with_position,
+            current_fts_format_version().posting_tail_codec(),
+            block_size,
+        )
+    }
+
+    pub fn new_with_posting_tail_codec_and_block_size(
+        with_position: bool,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Self {
+        validate_block_size(block_size).expect("invalid posting list block size");
         Self {
             with_positions: with_position,
             posting_tail_codec,
@@ -4181,6 +5454,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             len: 0,
             memory_size_bytes: 0,
         }
@@ -4202,8 +5476,8 @@ impl PostingListBuilder {
         &self,
         mut visit: impl FnMut(u32, u32, Option<Vec<u32>>) -> std::result::Result<(), E>,
     ) -> std::result::Result<(), E> {
-        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
-        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        let mut doc_ids = Vec::with_capacity(self.block_size);
+        let mut frequencies = Vec::with_capacity(self.block_size);
         let mut decoded_positions = Vec::new();
         let mut position_block_index = 0usize;
 
@@ -4211,7 +5485,12 @@ impl PostingListBuilder {
             for block in encoded_blocks.iter() {
                 doc_ids.clear();
                 frequencies.clear();
-                super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
+                super::encoding::decode_full_posting_block(
+                    block,
+                    &mut doc_ids,
+                    &mut frequencies,
+                    self.block_size,
+                );
                 decoded_positions.clear();
                 if self.with_positions {
                     let position_blocks = self
@@ -4291,7 +5570,7 @@ impl PostingListBuilder {
         }
         self.len += 1;
 
-        if self.tail_entries.len() == BLOCK_SIZE {
+        if self.tail_entries.len() == self.block_size {
             self.flush_tail_block()
                 .expect("posting list block compression should succeed");
         }
@@ -4350,7 +5629,7 @@ impl PostingListBuilder {
                 self.open_doc_id = None;
                 self.open_doc_frequency = 0;
                 self.open_doc_last_position = None;
-                if self.tail_entries.len() == BLOCK_SIZE {
+                if self.tail_entries.len() == self.block_size {
                     self.flush_tail_block()?;
                 }
                 Ok(())
@@ -4401,13 +5680,17 @@ impl PostingListBuilder {
             self.open_doc_id.is_none(),
             "cannot flush a posting block while a document is still open"
         );
-        debug_assert_eq!(self.tail_entries.len(), BLOCK_SIZE);
-        let mut doc_ids = [0u32; BLOCK_SIZE];
-        let mut frequencies = [0u32; BLOCK_SIZE];
-        for (index, entry) in self.tail_entries.iter().enumerate() {
-            doc_ids[index] = entry.doc_id;
-            frequencies[index] = entry.frequency;
-        }
+        debug_assert_eq!(self.tail_entries.len(), self.block_size);
+        let doc_ids = self
+            .tail_entries
+            .iter()
+            .map(|entry| entry.doc_id)
+            .collect::<Vec<_>>();
+        let frequencies = self
+            .tail_entries
+            .iter()
+            .map(|entry| entry.frequency)
+            .collect::<Vec<_>>();
         let encoded_blocks_size_before = self
             .encoded_blocks
             .as_ref()
@@ -4511,6 +5794,7 @@ impl PostingListBuilder {
     fn build_batch(
         self,
         compressed: LargeBinaryArray,
+        impacts: Option<ImpactSkipData>,
         max_score: f32,
         schema: SchemaRef,
         positions: Option<CompressedPositionStorage>,
@@ -4529,6 +5813,22 @@ impl PostingListBuilder {
                 length as u32,
             ))) as ArrayRef,
         ];
+        if schema.field_with_name(IMPACT_COL).is_ok() {
+            let impacts = impacts.ok_or_else(|| {
+                Error::index(format!(
+                    "impact column requested without impact data for posting length {}",
+                    length
+                ))
+            })?;
+            let impact_offsets =
+                OffsetBuffer::new(ScalarBuffer::from(vec![0, impacts.entries().len() as i32]));
+            columns.push(Arc::new(ListArray::try_new(
+                Arc::new(Field::new("item", datatypes::DataType::LargeBinary, true)),
+                impact_offsets,
+                Arc::new(impacts.entries().clone()),
+                None,
+            )?) as ArrayRef);
+        }
         columns.extend(Self::build_position_columns(positions)?);
 
         let batch = RecordBatch::try_new(schema, columns)?;
@@ -4578,6 +5878,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -4587,6 +5888,7 @@ impl PostingListBuilder {
         let parts = PostingListParts {
             with_positions,
             posting_tail_codec,
+            block_size,
             length: len as usize,
             encoded_blocks: encoded_blocks
                 .map(|encoded_blocks| *encoded_blocks)
@@ -4597,13 +5899,19 @@ impl PostingListBuilder {
             tail_entries: tail_entries.as_slice(),
             tail_position_block: with_positions.then(|| tail_positions.finish()),
         };
-        let (compressed, shared_positions, max_score) =
+        let (compressed, shared_positions, max_score, impacts) =
             Self::build_compressed_with_scores_from_parts(parts, docs)?;
         let positions = match legacy_positions {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        batch_builder.append(compressed, max_score, len, positions.as_ref())
+        batch_builder.append(
+            compressed,
+            Some(&impacts),
+            max_score,
+            len,
+            positions.as_ref(),
+        )
     }
 
     fn extend_tail_components(
@@ -4620,11 +5928,17 @@ impl PostingListBuilder {
     fn build_compressed_with_scores_from_parts(
         parts: PostingListParts<'_>,
         docs: &DocSet,
-    ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+    ) -> Result<(
+        LargeBinaryArray,
+        Option<SharedPositionStream>,
+        f32,
+        ImpactSkipData,
+    )> {
         let PostingListParts {
             with_positions,
             posting_tail_codec,
             length,
+            block_size,
             mut encoded_blocks,
             mut encoded_position_blocks,
             tail_entries,
@@ -4633,41 +5947,58 @@ impl PostingListBuilder {
         let avgdl = docs.average_length();
         let idf_scale = idf(length, docs.len()) * (K1 + 1.0);
         let mut max_score = f32::MIN;
-        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
-        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        let mut doc_ids = Vec::with_capacity(block_size);
+        let mut frequencies = Vec::with_capacity(block_size);
+        let mut impact_block = Vec::with_capacity(block_size);
+        let mut impact_builder =
+            ImpactSkipDataBuilder::with_capacity(length.div_ceil(block_size), block_size);
 
         for index in 0..encoded_blocks.len() {
             let block = encoded_blocks.block(index);
             doc_ids.clear();
             frequencies.clear();
-            super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
-            let block_score = compute_block_score(
+            super::encoding::decode_full_posting_block(
+                block,
+                &mut doc_ids,
+                &mut frequencies,
+                block_size,
+            );
+            let block_score = compute_block_score_and_impact_block(
                 docs,
                 avgdl,
                 idf_scale,
                 doc_ids.iter().copied(),
                 frequencies.iter().copied(),
+                &mut impact_block,
             );
+            impact_builder.append_block(impact_block.as_slice())?;
             max_score = max_score.max(block_score);
-            encoded_blocks.set_block_score(index, block_score);
+            if super::encoding::posting_block_score_prefix_len(block_size) > 0 {
+                encoded_blocks.set_block_score(index, block_score);
+            }
         }
 
         if !tail_entries.is_empty() {
             Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies);
-            let block_score = compute_block_score(
+            let block_score = compute_block_score_and_impact_block(
                 docs,
                 avgdl,
                 idf_scale,
                 doc_ids.iter().copied(),
                 frequencies.iter().copied(),
+                &mut impact_block,
             );
+            impact_builder.append_block(impact_block.as_slice())?;
             max_score = max_score.max(block_score);
             encoded_blocks.append_remainder_block_with_codec(
                 doc_ids.as_slice(),
                 frequencies.as_slice(),
                 posting_tail_codec,
+                block_size,
             )?;
-            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if super::encoding::posting_block_score_prefix_len(block_size) > 0 {
+                encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            }
             if with_positions {
                 encoded_position_blocks.push_encoded_block(
                     tail_position_block
@@ -4677,22 +6008,27 @@ impl PostingListBuilder {
             }
         }
 
+        let impacts = impact_builder.finish()?;
         Ok((
             encoded_blocks.into_array(),
             with_positions.then(|| encoded_position_blocks.into_stream()),
             max_score,
+            impacts,
         ))
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn build_compressed_with_block_scores_from_parts(
         with_positions: bool,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         mut encoded_blocks: EncodedBlocks,
         mut encoded_position_blocks: EncodedPositionBlocks,
         tail_entries: &[RawDocInfo],
         tail_position_block: Option<Vec<u8>>,
         mut block_max_scores: impl Iterator<Item = f32>,
     ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+        let has_score_prefix = super::encoding::posting_block_score_prefix_len(block_size) > 0;
         let mut max_score = f32::MIN;
         let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
         let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
@@ -4702,7 +6038,9 @@ impl PostingListBuilder {
                 .next()
                 .ok_or_else(|| Error::index("missing block max score".to_owned()))?;
             max_score = max_score.max(block_score);
-            encoded_blocks.set_block_score(index, block_score);
+            if has_score_prefix {
+                encoded_blocks.set_block_score(index, block_score);
+            }
         }
 
         if !tail_entries.is_empty() {
@@ -4715,8 +6053,11 @@ impl PostingListBuilder {
                 doc_ids.as_slice(),
                 frequencies.as_slice(),
                 posting_tail_codec,
+                block_size,
             )?;
-            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if has_score_prefix {
+                encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            }
             if with_positions {
                 encoded_position_blocks.push_encoded_block(
                     tail_position_block
@@ -4734,12 +6075,16 @@ impl PostingListBuilder {
     }
 
     pub fn to_batch(self, block_max_scores: Vec<f32>) -> Result<RecordBatch> {
-        let format_version = if self.posting_tail_codec == PostingTailCodec::Fixed32 {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
-        let schema = inverted_list_schema_for_version(self.has_positions(), format_version);
+        let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            self.posting_tail_codec,
+            self.block_size,
+        )?;
+        let schema = inverted_list_schema_for_version_with_block_size_and_impacts(
+            self.has_positions(),
+            format_version,
+            self.block_size,
+            false,
+        );
         let legacy_positions =
             if self.with_positions && !format_version.uses_shared_position_stream() {
                 Some(self.build_legacy_positions()?)
@@ -4756,6 +6101,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -4766,6 +6112,7 @@ impl PostingListBuilder {
             Self::build_compressed_with_block_scores_from_parts(
                 with_positions,
                 posting_tail_codec,
+                block_size,
                 encoded_blocks
                     .map(|encoded_blocks| *encoded_blocks)
                     .unwrap_or_default(),
@@ -4786,6 +6133,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             memory_size_bytes: 0,
             len,
         };
@@ -4793,17 +6141,11 @@ impl PostingListBuilder {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        builder.build_batch(compressed, max_score, schema, positions)
+        builder.build_batch(compressed, None, max_score, schema, positions)
     }
 
     pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result<RecordBatch> {
-        let format_version = if schema.column_with_name(POSITION_COL).is_some()
-            && schema.column_with_name(COMPRESSED_POSITION_COL).is_none()
-        {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
+        let format_version = parse_format_version_from_metadata(schema.metadata())?;
         let legacy_positions =
             if self.with_positions && !format_version.uses_shared_position_stream() {
                 Some(self.build_legacy_positions()?)
@@ -4820,6 +6162,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -4829,6 +6172,7 @@ impl PostingListBuilder {
         let parts = PostingListParts {
             with_positions,
             posting_tail_codec,
+            block_size,
             length: len as usize,
             encoded_blocks: encoded_blocks
                 .map(|encoded_blocks| *encoded_blocks)
@@ -4839,7 +6183,7 @@ impl PostingListBuilder {
             tail_entries: tail_entries.as_slice(),
             tail_position_block: with_positions.then(|| tail_positions.finish()),
         };
-        let (compressed, shared_positions, max_score) =
+        let (compressed, shared_positions, max_score, impacts) =
             Self::build_compressed_with_scores_from_parts(parts, docs)?;
         let builder = Self {
             with_positions,
@@ -4851,6 +6195,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             memory_size_bytes: 0,
             len,
         };
@@ -4858,13 +6203,16 @@ impl PostingListBuilder {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        builder.build_batch(compressed, max_score, schema, positions)
+        builder.build_batch(compressed, Some(impacts), max_score, schema, positions)
     }
 
     pub fn remap(&mut self, removed: &[u32]) {
         let mut cursor = 0;
-        let mut new_builder =
-            Self::new_with_posting_tail_codec(self.has_positions(), self.posting_tail_codec);
+        let mut new_builder = Self::new_with_posting_tail_codec_and_block_size(
+            self.has_positions(),
+            self.posting_tail_codec,
+            self.block_size,
+        );
         for (doc_id, freq, positions) in self.iter() {
             while cursor < removed.len() && removed[cursor] < doc_id {
                 cursor += 1;
@@ -4883,19 +6231,23 @@ impl PostingListBuilder {
     }
 }
 
-fn compute_block_score(
+fn compute_block_score_and_impact_block(
     docs: &DocSet,
     avgdl: f32,
     idf_scale: f32,
     doc_ids: impl Iterator<Item = u32>,
     frequencies: impl Iterator<Item = u32>,
+    impact_block: &mut Vec<(u32, u32, u32)>,
 ) -> f32 {
+    impact_block.clear();
     let mut block_max_score = f32::MIN;
     for (doc_id, freq) in doc_ids.zip(frequencies) {
-        let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl);
-        let freq = freq as f32;
-        let score = freq / (freq + doc_norm);
+        let doc_len = docs.num_tokens(doc_id);
+        let doc_norm = K1 * (1.0 - B + B * doc_len as f32 / avgdl);
+        let freq_f32 = freq as f32;
+        let score = freq_f32 / (freq_f32 + doc_norm);
         block_max_score = block_max_score.max(score);
+        impact_block.push((doc_id, freq, doc_len));
     }
     block_max_score * idf_scale
 }
@@ -5006,43 +6358,238 @@ impl Ord for RawDocInfo {
     }
 }
 
-// DocSet is a mapping from row ids to the number of tokens in the document
-// It's used to sort the documents by the bm25 score
-#[derive(Debug, Clone, Default, DeepSizeOf)]
-pub struct DocSet {
-    row_ids: Vec<u64>,
-    num_tokens: Vec<u32>,
-    // (row_id, doc_id) pairs sorted by row_id
-    inv: Vec<(u64, u32)>,
+/// Lucene SmallFloat-style document-length quantization for 256-document-block scoring and impact
+/// norms: a 4-mantissa-bit float-like byte code. Values 0-7 are exact; larger
+/// values keep their top four significand bits (relative error <= 6.25%) and
+/// decode to their bucket floor. The floor only ever shortens a doc, so impact
+/// bounds remain conservative for exact scoring as well as quantized scoring.
+pub(super) fn quantize_doc_length(value: u32) -> u8 {
+    let num_bits = 32 - value.leading_zeros();
+    if num_bits < 4 {
+        value as u8
+    } else {
+        let shift = num_bits - 4;
+        (((value >> shift) as u8) & 0x07) | (((shift + 1) as u8) << 3)
+    }
+}
 
-    total_tokens: u64,
+#[inline]
+pub(super) fn dequantize_doc_length(code: u8) -> u32 {
+    DEQUANTIZED_DOC_LENGTHS[code as usize]
 }
 
-impl DocSet {
-    #[inline]
-    pub fn len(&self) -> usize {
-        // Use num_tokens instead of row_ids so the deferred-row_ids
-        // scoring path (which constructs a DocSet via
-        // [`Self::from_num_tokens_only`]) still reports the right doc
-        // count.
-        self.num_tokens.len()
-    }
+pub(super) static DEQUANTIZED_DOC_LENGTHS: [u32; 256] = build_dequantized_doc_lengths();
 
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
+const fn build_dequantized_doc_lengths() -> [u32; 256] {
+    let mut table = [0u32; 256];
+    let mut code = 0usize;
+    while code < 256 {
+        let bits = (code & 0x07) as u64;
+        let shift = (code >> 3) as i64 - 1;
+        let decoded = if shift < 0 {
+            bits
+        } else {
+            (bits | 0x08) << shift
+        };
+        // Codes past the largest u32 encoding are never produced; saturate so
+        // the table stays total.
+        table[code] = if decoded > u32::MAX as u64 {
+            u32::MAX
+        } else {
+            decoded as u32
+        };
+        code += 1;
     }
+    table
+}
 
-    /// True iff the per-doc `row_id` array is populated. The
-    /// deferred-row_id scoring path constructs DocSets with the array
-    /// left empty so wand can skip the load; callers that need to do
-    /// row_id lookups in the inner loop must check this and fall back
-    /// to async resolution otherwise.
-    #[inline]
-    pub fn has_row_ids(&self) -> bool {
-        !self.row_ids.is_empty()
-    }
+#[derive(Debug, Clone)]
+enum NumTokens {
+    Owned(Vec<u32>),
+    Shared(ScalarBuffer<u32>),
+}
 
-    pub fn iter(&self) -> impl Iterator<Item = (&u64, &u32)> {
+impl Default for NumTokens {
+    fn default() -> Self {
+        Self::Owned(Vec::new())
+    }
+}
+
+impl std::ops::Deref for NumTokens {
+    type Target = [u32];
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values,
+        }
+    }
+}
+
+impl DeepSizeOf for NumTokens {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match self {
+            Self::Owned(values) => values.deep_size_of_children(context),
+            Self::Shared(values) => values.deep_size_of_children(context),
+        }
+    }
+}
+
+impl NumTokens {
+    fn with_capacity(capacity: usize) -> Self {
+        Self::Owned(Vec::with_capacity(capacity))
+    }
+
+    fn into_owned(self) -> Vec<u32> {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values.to_vec(),
+        }
+    }
+
+    fn push(&mut self, value: u32) {
+        match self {
+            Self::Owned(values) => values.push(value),
+            Self::Shared(values) => {
+                let mut owned = values.to_vec();
+                owned.push(value);
+                *self = Self::Owned(owned);
+            }
+        }
+    }
+
+    fn memory_size(&self) -> usize {
+        match self {
+            Self::Owned(values) => values.capacity() * std::mem::size_of::<u32>(),
+            Self::Shared(values) => values.inner().capacity(),
+        }
+    }
+}
+
+/// `Shared` is a zero-copy view of the partition's `DocRowIdsKey` cache
+/// entry; hold it only transiently, or evicting the entry can no longer
+/// free the memory.
+#[derive(Debug, Clone)]
+enum RowIds {
+    Owned(Vec<u64>),
+    Shared(ScalarBuffer<u64>),
+}
+
+impl Default for RowIds {
+    fn default() -> Self {
+        Self::Owned(Vec::new())
+    }
+}
+
+impl std::ops::Deref for RowIds {
+    type Target = [u64];
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values,
+        }
+    }
+}
+
+impl DeepSizeOf for RowIds {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match self {
+            Self::Owned(values) => values.deep_size_of_children(context),
+            // Weighed by the DocRowIdsKey cache entry.
+            Self::Shared(_) => 0,
+        }
+    }
+}
+
+impl RowIds {
+    fn with_capacity(capacity: usize) -> Self {
+        Self::Owned(Vec::with_capacity(capacity))
+    }
+
+    fn into_owned(self) -> Vec<u64> {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values.to_vec(),
+        }
+    }
+
+    fn push(&mut self, value: u64) {
+        match self {
+            Self::Owned(values) => values.push(value),
+            Self::Shared(values) => {
+                let mut owned = values.to_vec();
+                owned.push(value);
+                *self = Self::Owned(owned);
+            }
+        }
+    }
+
+    fn memory_size(&self) -> usize {
+        match self {
+            Self::Owned(values) => values.capacity() * std::mem::size_of::<u64>(),
+            Self::Shared(_) => 0,
+        }
+    }
+}
+
+// DocSet is a mapping from row ids to the number of tokens in the document
+// It's used to sort the documents by the bm25 score
+#[derive(Debug, Clone, Default)]
+pub struct DocSet {
+    row_ids: RowIds,
+    num_tokens: NumTokens,
+    // (row_id, doc_id) pairs sorted by row_id
+    inv: Vec<(u64, u32)>,
+
+    total_tokens: u64,
+
+    // 256-document-block partitions score with quantized document lengths: the
+    // flag is set at partition load and the byte-norm slab bakes lazily on
+    // first scoring use (shared by clones of the loaded set). 128-block
+    // partitions never set the flag and keep exact scoring.
+    scoring_quantized: bool,
+    norms: Arc<std::sync::OnceLock<Box<[u8]>>>,
+}
+
+impl DeepSizeOf for DocSet {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.row_ids.deep_size_of_children(context)
+            + self.num_tokens.deep_size_of_children(context)
+            + self.inv.deep_size_of_children(context)
+            + self
+                .norms
+                .get()
+                .map(|slab| std::mem::size_of_val(slab.as_ref()))
+                .unwrap_or(0)
+    }
+}
+
+impl DocSet {
+    #[inline]
+    pub fn len(&self) -> usize {
+        // Use num_tokens instead of row_ids so the deferred-row_ids
+        // scoring path (which constructs a DocSet via
+        // [`Self::from_num_tokens_only`]) still reports the right doc
+        // count.
+        self.num_tokens.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// True iff the per-doc `row_id` array is populated. The
+    /// deferred-row_id scoring path constructs DocSets with the array
+    /// left empty so wand can skip the load; callers that need to do
+    /// row_id lookups in the inner loop must check this and fall back
+    /// to async resolution otherwise.
+    #[inline]
+    pub fn has_row_ids(&self) -> bool {
+        !self.row_ids.is_empty()
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&u64, &u32)> {
         self.row_ids.iter().zip(self.num_tokens.iter())
     }
 
@@ -5052,12 +6599,11 @@ impl DocSet {
 
     /// Resolve a `row_id` to every `doc_id` it owns.
     ///
-    /// A scalar column maps each row to a single document, but a
-    /// `list<string>` column indexes every element as its own document, so a
-    /// single `row_id` can own several `doc_id`s sharing that key in `inv`.
+    /// Modern indexes map each row to a single document. Older list indexes
+    /// may have indexed each list element as its own document, so a single
+    /// `row_id` can still own several `doc_id`s sharing that key in `inv`.
     /// The prefilter path (`flat_search`) walks an allow-list of row_ids and
-    /// must evaluate *all* of a row's documents; resolving to one `doc_id`
-    /// silently drops matches at non-last list positions (lancedb#3352).
+    /// must evaluate all legacy documents for that row.
     pub fn doc_ids(&self, row_id: u64) -> impl Iterator<Item = u64> + '_ {
         if self.inv.is_empty() {
             // in legacy format, the row id is doc id (one document per row)
@@ -5085,9 +6631,19 @@ impl DocSet {
         doc_ids: impl Iterator<Item = &'a u32>,
         freqs: impl Iterator<Item = &'a u32>,
     ) -> Vec<f32> {
+        self.calculate_block_max_scores_with_block_size(doc_ids, freqs, LEGACY_BLOCK_SIZE)
+    }
+
+    pub fn calculate_block_max_scores_with_block_size<'a>(
+        &self,
+        doc_ids: impl Iterator<Item = &'a u32>,
+        freqs: impl Iterator<Item = &'a u32>,
+        block_size: usize,
+    ) -> Vec<f32> {
+        validate_block_size(block_size).expect("invalid posting list block size");
         let avgdl = self.average_length();
         let length = doc_ids.size_hint().0;
-        let num_blocks = length.div_ceil(BLOCK_SIZE);
+        let num_blocks = length.div_ceil(block_size);
         let mut block_max_scores = Vec::with_capacity(num_blocks);
         let idf_scale = idf(length, self.len()) * (K1 + 1.0);
         let mut max_score = f32::MIN;
@@ -5098,13 +6654,13 @@ impl DocSet {
             if score > max_score {
                 max_score = score;
             }
-            if (i + 1) % BLOCK_SIZE == 0 {
+            if (i + 1) % block_size == 0 {
                 max_score *= idf_scale;
                 block_max_scores.push(max_score);
                 max_score = f32::MIN;
             }
         }
-        if !length.is_multiple_of(BLOCK_SIZE) {
+        if !length.is_multiple_of(block_size) {
             max_score *= idf_scale;
             block_max_scores.push(max_score);
         }
@@ -5147,14 +6703,60 @@ impl DocSet {
     /// `num_tokens_by_row_id` calls, and the per-partition caller
     /// resolves doc_id → row_id for the surviving top-K post-wand.
     pub fn from_num_tokens_only(num_tokens_col: &arrow_array::UInt32Array) -> Self {
-        let num_tokens = num_tokens_col.values().to_vec();
-        let total_tokens = num_tokens.iter().map(|&n| n as u64).sum();
+        let total_tokens = num_tokens_col.values().iter().map(|&n| n as u64).sum();
+        Self::from_cached_num_tokens(num_tokens_col, total_tokens)
+    }
+
+    /// Build a zero-copy num-tokens-only view from an Arrow column and its
+    /// already-computed total. The caller must guarantee that `total_tokens`
+    /// is the sum of `num_tokens_col`.
+    pub(crate) fn from_cached_num_tokens(
+        num_tokens_col: &arrow_array::UInt32Array,
+        total_tokens: u64,
+    ) -> Self {
         Self {
-            row_ids: Vec::new(),
-            num_tokens,
+            row_ids: RowIds::default(),
+            num_tokens: NumTokens::Shared(num_tokens_col.values().clone()),
             inv: Vec::new(),
             total_tokens,
+            scoring_quantized: false,
+            norms: Arc::new(std::sync::OnceLock::new()),
+        }
+    }
+
+    /// Per-query view for the masked wand path: this num-tokens-only set
+    /// plus a transient borrow of the row-ids cache entry.
+    pub(crate) fn with_shared_row_ids(&self, row_ids: ScalarBuffer<u64>) -> Self {
+        let mut docs = self.clone();
+        docs.row_ids = RowIds::Shared(row_ids);
+        docs
+    }
+
+    /// Resident scoring set for a modern partition: shared `num_tokens` plus
+    /// `inv`; no row_ids — the column lives only in the cache entry.
+    pub(crate) fn from_cached_num_tokens_with_inv(
+        row_ids_col: &UInt64Array,
+        num_tokens_col: &arrow_array::UInt32Array,
+        total_tokens: u64,
+    ) -> Self {
+        let row_ids = row_ids_col.values();
+        let mut inv: Vec<(u64, u32)> = row_ids
+            .iter()
+            .enumerate()
+            .map(|(doc_id, row_id)| (*row_id, doc_id as u32))
+            .collect();
+        if !row_ids.is_sorted() {
+            inv.sort_unstable_by_key(|entry| entry.0);
         }
+        let mut docs = Self::from_cached_num_tokens(num_tokens_col, total_tokens);
+        docs.inv = inv;
+        docs
+    }
+
+    /// True iff `doc_ids()` can answer reverse lookups; flat_search requires
+    /// this.
+    pub(crate) fn supports_reverse_lookup(&self) -> bool {
+        !self.inv.is_empty() || self.has_row_ids()
     }
 
     /// Build a `DocSet` from already-loaded `row_id` and `num_tokens`
@@ -5185,10 +6787,12 @@ impl DocSet {
 
             let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
             return Ok(Self {
-                row_ids,
-                num_tokens,
+                row_ids: RowIds::Owned(row_ids),
+                num_tokens: NumTokens::Owned(num_tokens),
                 inv: Vec::new(),
                 total_tokens,
+                scoring_quantized: false,
+                norms: Arc::new(std::sync::OnceLock::new()),
             });
         }
 
@@ -5226,10 +6830,12 @@ impl DocSet {
 
             let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
             return Ok(Self {
-                row_ids,
-                num_tokens,
+                row_ids: RowIds::Owned(row_ids),
+                num_tokens: NumTokens::Owned(num_tokens),
                 inv,
                 total_tokens,
+                scoring_quantized: false,
+                norms: Arc::new(std::sync::OnceLock::new()),
             });
         }
 
@@ -5245,10 +6851,12 @@ impl DocSet {
         }
         let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
         Ok(Self {
-            row_ids,
-            num_tokens,
+            row_ids: RowIds::Owned(row_ids),
+            num_tokens: NumTokens::Owned(num_tokens),
             inv,
             total_tokens,
+            scoring_quantized: false,
+            norms: Arc::new(std::sync::OnceLock::new()),
         })
     }
 
@@ -5257,8 +6865,10 @@ impl DocSet {
     pub fn remap(&mut self, mapping: &RowAddrRemap) -> Vec<u32> {
         let mut removed = Vec::new();
         let len = self.len();
-        let row_ids = std::mem::replace(&mut self.row_ids, Vec::with_capacity(len));
-        let num_tokens = std::mem::replace(&mut self.num_tokens, Vec::with_capacity(len));
+        let row_ids = std::mem::replace(&mut self.row_ids, RowIds::with_capacity(len)).into_owned();
+        let num_tokens =
+            std::mem::replace(&mut self.num_tokens, NumTokens::with_capacity(len)).into_owned();
+        self.invalidate_norms();
         self.total_tokens = 0;
         for (doc_id, (row_id, num_token)) in std::iter::zip(row_ids, num_tokens).enumerate() {
             match mapping.get(row_id) {
@@ -5285,6 +6895,39 @@ impl DocSet {
         self.num_tokens[doc_id as usize]
     }
 
+    /// Enable quantized document-length scoring for 256-document-block partitions.
+    pub fn set_quantized_scoring(&mut self, quantized: bool) {
+        self.scoring_quantized = quantized;
+    }
+
+    /// The quantized document-length slab when this set scores quantized,
+    /// baked on first use; `None` for exact-scoring sets.
+    pub fn scoring_norms(&self) -> Option<&[u8]> {
+        if !self.scoring_quantized {
+            return None;
+        }
+        Some(
+            self.norms
+                .get_or_init(|| {
+                    self.num_tokens
+                        .iter()
+                        .map(|&n| quantize_doc_length(n))
+                        .collect()
+                })
+                .as_ref(),
+        )
+    }
+
+    /// Document length as scoring sees it: the quantized bucket floor for
+    /// 256-document-block partitions, the exact value otherwise.
+    #[inline]
+    pub fn scoring_num_tokens(&self, doc_id: u32) -> u32 {
+        match self.scoring_norms() {
+            Some(norms) => dequantize_doc_length(norms[doc_id as usize]),
+            None => self.num_tokens[doc_id as usize],
+        }
+    }
+
     // this can be used only if it's a legacy format,
     // which store the sorted row ids so that we can use binary search
     #[inline]
@@ -5301,12 +6944,21 @@ impl DocSet {
         self.row_ids.push(row_id);
         self.num_tokens.push(num_tokens);
         self.total_tokens += num_tokens as u64;
+        self.invalidate_norms();
         self.row_ids.len() as u32 - 1
     }
 
+    // Drop the baked norm slab after a mutation; it re-bakes on the next
+    // scoring use.
+    fn invalidate_norms(&mut self) {
+        if self.norms.get().is_some() {
+            self.norms = Arc::new(std::sync::OnceLock::new());
+        }
+    }
+
     pub(crate) fn memory_size(&self) -> usize {
-        self.row_ids.capacity() * std::mem::size_of::<u64>()
-            + self.num_tokens.capacity() * std::mem::size_of::<u32>()
+        self.row_ids.memory_size()
+            + self.num_tokens.memory_size()
             + self.inv.capacity() * std::mem::size_of::<(u64, u32)>()
     }
 }
@@ -5330,6 +6982,12 @@ pub fn flat_full_text_search(
     match batches[0][doc_col].data_type() {
         DataType::Utf8 => do_flat_full_text_search::<i32>(batches, doc_col, query, tokenizer),
         DataType::LargeUtf8 => do_flat_full_text_search::<i64>(batches, doc_col, query, tokenizer),
+        DataType::List(_) => {
+            do_flat_full_text_search_list::<i32>(batches, doc_col, query, tokenizer)
+        }
+        DataType::LargeList(_) => {
+            do_flat_full_text_search_list::<i64>(batches, doc_col, query, tokenizer)
+        }
         data_type => Err(Error::invalid_input(format!(
             "unsupported data type {} for inverted index",
             data_type
@@ -5365,6 +7023,46 @@ fn do_flat_full_text_search<Offset: OffsetSizeTrait>(
     Ok(results)
 }
 
+fn do_flat_full_text_search_list<ListOffset: OffsetSizeTrait>(
+    batches: &[&RecordBatch],
+    doc_col: &str,
+    query: &str,
+    tokenizer: Option<Box<dyn LanceTokenizer>>,
+) -> Result<Vec<u64>> {
+    let mut results = Vec::new();
+    let mut tokenizer =
+        tokenizer.unwrap_or_else(|| InvertedIndexParams::default().build().unwrap());
+    let query_tokens = collect_query_tokens(query, &mut tokenizer);
+
+    for batch in batches {
+        let row_id_array = batch[ROW_ID].as_primitive::<UInt64Type>();
+        let doc_array = batch[doc_col].as_list::<ListOffset>();
+        match doc_array.value_type() {
+            DataType::Utf8 | DataType::LargeUtf8 => {}
+            data_type => {
+                return Err(Error::invalid_input(format!(
+                    "unsupported list item data type {} for inverted index",
+                    data_type
+                )));
+            }
+        }
+        for i in 0..row_id_array.len() {
+            if doc_array.is_null(i) {
+                continue;
+            }
+            let elements = doc_array.value(i);
+            if iter_str_array(elements.as_ref())
+                .flatten()
+                .any(|element| has_query_token(element, &mut tokenizer, &query_tokens))
+            {
+                results.push(row_id_array.value(i));
+            }
+        }
+    }
+
+    Ok(results)
+}
+
 const FLAT_ROW_ID_COL_IDX: usize = 0;
 const FLAT_ALL_TOKENS_COL_IDX: usize = 1;
 const FLAT_QUERY_TOKEN_COUNTS_COL_IDX: usize = 2;
@@ -5405,6 +7103,7 @@ async fn tokenize_and_count(
         ),
     ]));
     let output_schema_clone = output_schema.clone();
+    let query_token_indices = Arc::new(query_token_indices(query_tokens.as_ref()));
     let bytes_accumulated = Arc::new(AtomicU64::new(0));
     let bytes_warning_emitted = Arc::new(AtomicBool::new(false));
 
@@ -5413,6 +7112,7 @@ async fn tokenize_and_count(
             let mut tokenizer = tokenizer.box_clone();
             let output_schema = output_schema.clone();
             let query_tokens = query_tokens.clone();
+            let query_token_indices = query_token_indices.clone();
             let bytes_accumulated = bytes_accumulated.clone();
             let bytes_warning_emitted = bytes_warning_emitted.clone();
             let elapsed_compute = elapsed_compute.clone();
@@ -5422,6 +7122,8 @@ async fn tokenize_and_count(
                 // thread is invisible to the caller's poll timer otherwise).
                 let start = std::time::Instant::now();
                 let batch = batch?;
+                let row_id_array = batch[ROW_ID].as_primitive::<UInt64Type>();
+                let mut row_ids = UInt64Builder::with_capacity(batch.num_rows());
                 let mut all_token_counts = UInt64Builder::with_capacity(batch.num_rows());
                 let mut query_token_counts = FixedSizeListBuilder::with_capacity(
                     UInt64Builder::with_capacity(batch.num_rows() * query_tokens.len()),
@@ -5429,42 +7131,79 @@ async fn tokenize_and_count(
                     batch.num_rows(),
                 );
                 let mut temp_query_token_counts = Vec::with_capacity(query_tokens.len());
-                let doc_iter = iter_str_array(batch.column(doc_col_idx));
-                for doc in doc_iter {
-                    let Some(doc) = doc else {
-                        all_token_counts.append_value(0);
-                        query_token_counts
-                            .values()
-                            .append_value_n(0, query_tokens.len());
-                        query_token_counts.append(true);
-                        continue;
-                    };
-
-                    temp_query_token_counts.clear();
-                    temp_query_token_counts.extend(std::iter::repeat_n(0, query_tokens.len()));
-
+                let mut count_text = |doc: &str, temp_query_token_counts: &mut Vec<u64>| -> u64 {
                     let mut stream = tokenizer.token_stream_for_doc(doc);
                     let mut all_tokens = 0;
                     while let Some(token) = stream.next() {
                         all_tokens += 1;
-                        if let Some(token_index) = query_tokens.token_index(&token.text) {
-                            temp_query_token_counts[token_index] += 1;
+                        if let Some(token_indices) = query_token_indices.get(&token.text) {
+                            for token_index in token_indices {
+                                temp_query_token_counts[*token_index] += 1;
+                            }
+                        }
+                    }
+                    all_tokens
+                };
+                let mut append_counts =
+                    |row_id: u64, all_tokens: u64, temp_query_token_counts: &[u64]| {
+                        row_ids.append_value(row_id);
+                        all_token_counts.append_value(all_tokens);
+                        for count in temp_query_token_counts.iter().copied() {
+                            query_token_counts.values().append_value(count);
+                        }
+                        query_token_counts.append(true);
+                    };
+                match batch.column(doc_col_idx).data_type() {
+                    DataType::Utf8 | DataType::LargeUtf8 => {
+                        let doc_iter = iter_str_array(batch.column(doc_col_idx));
+                        for (doc, row_id) in doc_iter.zip(row_id_array.values().iter()) {
+                            temp_query_token_counts.clear();
+                            temp_query_token_counts
+                                .extend(std::iter::repeat_n(0, query_tokens.len()));
+
+                            let Some(doc) = doc else {
+                                continue;
+                            };
+
+                            let all_tokens = count_text(doc, &mut temp_query_token_counts);
+                            if all_tokens > 0 {
+                                append_counts(*row_id, all_tokens, &temp_query_token_counts);
+                            }
                         }
                     }
-                    all_token_counts.append_value(all_tokens);
-                    for count in temp_query_token_counts.iter().copied() {
-                        query_token_counts.values().append_value(count);
+                    DataType::List(_) => {
+                        tokenize_and_count_list::<i32>(
+                            batch.column(doc_col_idx),
+                            row_id_array,
+                            &mut count_text,
+                            &mut append_counts,
+                            &mut temp_query_token_counts,
+                            query_tokens.len(),
+                        )?;
+                    }
+                    DataType::LargeList(_) => {
+                        tokenize_and_count_list::<i64>(
+                            batch.column(doc_col_idx),
+                            row_id_array,
+                            &mut count_text,
+                            &mut append_counts,
+                            &mut temp_query_token_counts,
+                            query_tokens.len(),
+                        )?;
+                    }
+                    data_type => {
+                        return DataFusionResult::Err(datafusion_common::DataFusionError::Execution(
+                            format!("unsupported data type {} for flat full text search", data_type),
+                        ));
                     }
-                    query_token_counts.append(true);
                 }
-                let row_ids = batch[ROW_ID].clone();
+                let row_ids = row_ids.finish();
                 let all_token_counts = all_token_counts.finish();
                 let query_token_counts = query_token_counts.finish();
                 let result_batch = RecordBatch::try_new(
-
                     output_schema,
                     vec![
-                        row_ids,
+                        Arc::new(row_ids) as ArrayRef,
                         Arc::new(all_token_counts) as ArrayRef,
                         Arc::new(query_token_counts) as ArrayRef,
                     ],
@@ -5490,6 +7229,58 @@ async fn tokenize_and_count(
     )?)
 }
 
+fn tokenize_and_count_list<ListOffset: OffsetSizeTrait>(
+    doc_col: &ArrayRef,
+    row_id_array: &arrow_array::PrimitiveArray<UInt64Type>,
+    count_text: &mut impl FnMut(&str, &mut Vec<u64>) -> u64,
+    append_counts: &mut impl FnMut(u64, u64, &[u64]),
+    temp_query_token_counts: &mut Vec<u64>,
+    query_tokens_len: usize,
+) -> DataFusionResult<()> {
+    let doc_array = doc_col.as_list::<ListOffset>();
+    match doc_array.value_type() {
+        DataType::Utf8 | DataType::LargeUtf8 => {}
+        data_type => {
+            return Err(datafusion_common::DataFusionError::Execution(format!(
+                "unsupported list item data type {} for flat full text search",
+                data_type
+            )));
+        }
+    }
+
+    for i in 0..row_id_array.len() {
+        if doc_array.is_null(i) {
+            continue;
+        }
+
+        temp_query_token_counts.clear();
+        temp_query_token_counts.extend(std::iter::repeat_n(0, query_tokens_len));
+
+        let elements = doc_array.value(i);
+        let mut all_tokens = 0;
+        for element in iter_str_array(elements.as_ref()).flatten() {
+            all_tokens += count_text(element, temp_query_token_counts);
+        }
+
+        if all_tokens > 0 {
+            append_counts(row_id_array.value(i), all_tokens, temp_query_token_counts);
+        }
+    }
+
+    Ok(())
+}
+
+fn query_token_indices(query_tokens: &Tokens) -> HashMap<String, Vec<usize>> {
+    let mut indices = HashMap::new();
+    for idx in 0..query_tokens.len() {
+        indices
+            .entry(query_tokens.get_token(idx).to_string())
+            .or_insert_with(Vec::new)
+            .push(idx);
+    }
+    indices
+}
+
 /// Initialize the BM25 scorer
 ///
 /// In order to calculate BM25 scores we need to know token counts for the entire corpus.  We extract these from the
@@ -5530,7 +7321,9 @@ fn initialize_scorer(
 
     for _ in 0..counted_input.num_rows() {
         for token_count in all_token_counts.iter_mut() {
-            *token_count += input_token_counters.next().unwrap_or_default();
+            if input_token_counters.next().unwrap_or_default() > 0 {
+                *token_count += 1;
+            }
         }
     }
 
@@ -5551,9 +7344,11 @@ fn flat_bm25_score(
     query_tokens: &Tokens,
     counted_input: &RecordBatch,
     scorer: &MemBM25Scorer,
+    operator: Operator,
 ) -> Result<RecordBatch> {
     let mut row_ids_builder = UInt64Builder::with_capacity(counted_input.num_rows());
     let mut scores_builder = Float32Builder::with_capacity(counted_input.num_rows());
+    let query_groups = query_position_groups(query_tokens);
 
     let mut row_ids_iter = counted_input
         .column(FLAT_ROW_ID_COL_IDX)
@@ -5578,16 +7373,24 @@ fn flat_bm25_score(
     for _ in 0..counted_input.num_rows() {
         let num_tokens_in_doc = all_token_counts_iter.next().expect_ok()?;
         let row_id = row_ids_iter.next().expect_ok()?;
+        let mut query_token_counts = Vec::with_capacity(query_tokens.len());
+        for _ in query_tokens {
+            query_token_counts.push(query_token_counts_iter.next().expect_ok()?);
+        }
         if num_tokens_in_doc == 0 {
-            for _ in query_tokens {
-                query_token_counts_iter.next().expect_ok()?;
-            }
+            continue;
+        }
+        if operator == Operator::And
+            && !query_groups
+                .iter()
+                .all(|group| group.iter().any(|idx| query_token_counts[*idx] > 0))
+        {
             continue;
         }
         let doc_norm = K1 * (1.0 - B + B * num_tokens_in_doc as f32 / scorer.avg_doc_length());
         let mut score = 0.0;
-        for token in query_tokens {
-            let freq = query_token_counts_iter.next().expect_ok()? as f32;
+        for (token, freq) in query_tokens.into_iter().zip(query_token_counts) {
+            let freq = freq as f32;
             let idf = idf(scorer.num_docs_containing_token(token), scorer.num_docs());
             score += idf * (freq * (K1 + 1.0) / (freq + doc_norm));
         }
@@ -5606,6 +7409,23 @@ fn flat_bm25_score(
     Ok(batch)
 }
 
+fn query_position_groups(query_tokens: &Tokens) -> Vec<Vec<usize>> {
+    let mut groups = Vec::new();
+    let mut current_position = None;
+    for idx in 0..query_tokens.len() {
+        let position = query_tokens.position(idx);
+        if current_position != Some(position) {
+            current_position = Some(position);
+            groups.push(Vec::new());
+        }
+        groups
+            .last_mut()
+            .expect("a group should exist after pushing for position")
+            .push(idx);
+    }
+    groups
+}
+
 #[deprecated(
     note = "use `flat_bm25_search_stream_with_metrics` to record CPU compute \
             time on a metric handle; pass `None` for the old behavior"
@@ -5643,6 +7463,58 @@ pub async fn flat_bm25_search_stream_with_metrics(
     base_scorer: Option<MemBM25Scorer>,
     target_batch_size: usize,
     elapsed_compute: Option<Time>,
+) -> DataFusionResult<SendableRecordBatchStream> {
+    flat_bm25_search_stream_with_metrics_and_operator(
+        input,
+        doc_col,
+        query,
+        tokenizer,
+        base_scorer,
+        target_batch_size,
+        Operator::Or,
+        elapsed_compute,
+    )
+    .await
+}
+
+/// Same as [`flat_bm25_search_stream_with_metrics`] but applies the provided
+/// match operator when deciding whether a flat-scanned row is a hit.
+///
+/// # Examples
+///
+/// ```no_run
+/// # async fn example(
+/// #     input: datafusion::execution::SendableRecordBatchStream,
+/// # ) -> Result<(), Box<dyn std::error::Error>> {
+/// use lance_index::scalar::inverted::{
+///     flat_bm25_search_stream_with_metrics_and_operator, query::Operator, InvertedIndexParams,
+/// };
+///
+/// let tokenizer = InvertedIndexParams::code().build()?;
+/// let _stream = flat_bm25_search_stream_with_metrics_and_operator(
+///     input,
+///     "code".to_string(),
+///     "Result".to_string(),
+///     tokenizer,
+///     None,
+///     1024,
+///     Operator::And,
+///     None,
+/// )
+/// .await?;
+/// # Ok(())
+/// # }
+/// ```
+#[allow(clippy::too_many_arguments)]
+pub async fn flat_bm25_search_stream_with_metrics_and_operator(
+    input: SendableRecordBatchStream,
+    doc_col: String,
+    query: String,
+    tokenizer: Box<dyn LanceTokenizer>,
+    base_scorer: Option<MemBM25Scorer>,
+    target_batch_size: usize,
+    operator: Operator,
+    elapsed_compute: Option<Time>,
 ) -> DataFusionResult<SendableRecordBatchStream> {
     let mut tokenizer = tokenizer;
 
@@ -5698,7 +7570,7 @@ pub async fn flat_bm25_search_stream_with_metrics(
     // All post-await work is synchronous; time the scorer + score + slicing loop together.
     let post_await_start = std::time::Instant::now();
     let scorer = initialize_scorer(base_scorer.as_ref(), query_tokens.as_ref(), &counted_input);
-    let scores = flat_bm25_score(query_tokens.as_ref(), &counted_input, &scorer)?;
+    let scores = flat_bm25_score(query_tokens.as_ref(), &counted_input, &scorer, operator)?;
 
     // Finally we emit batches according to the target batch size
     let num_out_batches = scores.num_rows().div_ceil(target_batch_size);
@@ -5734,7 +7606,10 @@ mod tests {
     use crate::prefilter::NoFilter;
     use crate::scalar::ScalarIndex;
     use crate::scalar::inverted::builder::{
-        InnerBuilder, InvertedIndexBuilder, PositionRecorder, inverted_list_schema,
+        InnerBuilder, InvertedIndexBuilder, PositionRecorder, doc_file_path, inverted_list_schema,
+        inverted_list_schema_for_version_with_block_size,
+        inverted_list_schema_for_version_with_block_size_and_impacts, posting_file_path,
+        token_file_path,
     };
     use crate::scalar::inverted::encoding::{
         compress_positions, compress_posting_list_with_tail_codec,
@@ -5742,35 +7617,222 @@ mod tests {
     };
     use crate::scalar::inverted::query::{FtsSearchParams, Operator};
     use crate::scalar::lance_format::LanceIndexStore;
-    use arrow::array::{AsArray, Int32Builder, LargeBinaryBuilder, ListBuilder, UInt32Builder};
+    use arrow::array::{
+        AsArray, GenericListBuilder, GenericStringBuilder, Int32Builder, LargeBinaryBuilder,
+        ListBuilder, UInt32Builder,
+    };
     use arrow::datatypes::{Float32Type, UInt32Type};
     use arrow_array::{ArrayRef, Float32Array, RecordBatch, StringArray, UInt32Array, UInt64Array};
     use arrow_schema::{DataType, Field, Schema};
     use std::collections::HashMap;
     use std::sync::Arc;
-    use std::sync::atomic::Ordering;
+    use std::sync::atomic::{AtomicU32, Ordering};
 
+    use crate::scalar::inverted::lazy_docset::DocRowIdsKey;
     use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer;
     use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer};
 
     use super::*;
 
-    async fn write_single_partition_index(
+    /// Test helper: resolve any `Pending` candidates a partition-level wand
+    /// walk emitted, so per-partition results can be asserted on real
+    /// row_ids. Production resolves only global top-k survivors inside
+    /// `InvertedIndex::bm25_search`.
+    async fn resolve_deferred_candidates(
+        docs: &LazyDocSet,
+        candidates: &mut [DocCandidate],
+    ) -> Result<()> {
+        let pending: Vec<u32> = candidates
+            .iter()
+            .filter_map(|c| match c.addr {
+                CandidateAddr::Pending(d) => Some(d),
+                CandidateAddr::RowId(_) => None,
+            })
+            .collect();
+        if pending.is_empty() {
+            return Ok(());
+        }
+        let mut iter = docs.resolve_row_ids(&pending).await?.into_iter();
+        for c in candidates {
+            if matches!(c.addr, CandidateAddr::Pending(_)) {
+                let r = iter.next().ok_or_else(|| {
+                    Error::internal("resolve_row_ids returned fewer items than requested")
+                })?;
+                c.addr = CandidateAddr::RowId(r);
+            }
+        }
+        Ok(())
+    }
+
+    #[derive(Debug)]
+    struct MetadataAccessDeniedStore {
+        inner: Arc<dyn IndexStore>,
+    }
+
+    impl DeepSizeOf for MetadataAccessDeniedStore {
+        fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+            self.inner.deep_size_of_children(context)
+        }
+    }
+
+    #[async_trait]
+    impl IndexStore for MetadataAccessDeniedStore {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.clone(),
+            })
+        }
+
+        fn io_parallelism(&self) -> usize {
+            self.inner.io_parallelism()
+        }
+
+        async fn new_index_file(
+            &self,
+            name: &str,
+            schema: Arc<Schema>,
+        ) -> Result<Box<dyn crate::scalar::IndexWriter>> {
+            self.inner.new_index_file(name, schema).await
+        }
+
+        async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
+            if name == METADATA_FILE {
+                Err(Error::io("metadata access denied"))
+            } else {
+                self.inner.open_index_file(name).await
+            }
+        }
+
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.with_io_priority(io_priority),
+            })
+        }
+
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner.copy_index_file(name, dest_store).await
+        }
+
+        async fn rename_index_file(
+            &self,
+            name: &str,
+            new_name: &str,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner.rename_index_file(name, new_name).await
+        }
+
+        async fn delete_index_file(&self, name: &str) -> Result<()> {
+            self.inner.delete_index_file(name).await
+        }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<crate::scalar::IndexFile>> {
+            self.inner.list_files_with_sizes().await
+        }
+    }
+
+    #[tokio::test]
+    async fn params_legacy_fallback_probes_tokens_after_metadata_access_denied() {
+        let tmpdir = TempObjDir::default();
+        let inner: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected = InvertedIndexParams::default();
+        let metadata = HashMap::from([(
+            "tokenizer".to_owned(),
+            serde_json::to_string(&expected).unwrap(),
+        )]);
+        let mut writer = inner
+            .new_index_file(TOKENS_FILE, Arc::new(Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+        let store = MetadataAccessDeniedStore { inner };
+
+        let actual = InvertedIndex::load_params(&store).await.unwrap();
+        assert_eq!(
+            serde_json::to_value(actual).unwrap(),
+            serde_json::to_value(expected).unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn params_legacy_probe_preserves_metadata_error_when_tokens_are_missing() {
+        let tmpdir = TempObjDir::default();
+        let inner: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let store = MetadataAccessDeniedStore { inner };
+
+        let error = InvertedIndex::load_params(&store).await.unwrap_err();
+        assert!(matches!(error, Error::IO { .. }));
+        assert!(error.to_string().contains("metadata access denied"));
+    }
+
+    #[tokio::test]
+    async fn params_metadata_ignores_unknown_fields() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected = InvertedIndexParams::default();
+        let mut params = serde_json::to_value(&expected).unwrap();
+        let params = params.as_object_mut().unwrap();
+        params.insert("skip_merge".to_owned(), true.into());
+        params.insert(
+            "future_parameter".to_owned(),
+            serde_json::json!({ "enabled": true }),
+        );
+        let metadata =
+            HashMap::from([("params".to_owned(), serde_json::to_string(params).unwrap())]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let actual = InvertedIndex::load_params(store.as_ref()).await.unwrap();
+        assert_eq!(
+            serde_json::to_value(actual).unwrap(),
+            serde_json::to_value(expected).unwrap()
+        );
+    }
+
+    async fn write_single_partition_index(
         store: Arc<LanceIndexStore>,
         params: InvertedIndexParams,
         token_set_format: TokenSetFormat,
         token: &str,
         row_id: u64,
     ) -> Result<Arc<InvertedIndex>> {
-        let mut partition = InnerBuilder::new_with_format_version(
+        let block_size = params.posting_block_size();
+        let format_version = params.resolved_format_version();
+        let mut partition = InnerBuilder::new_with_format_version_and_block_size(
             0,
             false,
             token_set_format,
-            InvertedListFormatVersion::V1,
+            format_version,
+            block_size,
         );
         partition.tokens.add(token.to_owned());
-        let mut posting_list =
-            PostingListBuilder::new_with_posting_tail_codec(false, PostingTailCodec::Fixed32);
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            block_size,
+        );
         posting_list.add(0, PositionRecorder::Count(1));
         partition.posting_lists.push(posting_list);
         partition.docs.append(row_id, 1);
@@ -5786,6 +7848,15 @@ mod tests {
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 token_set_format.to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                format_version.posting_tail_codec().as_str().to_owned(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
         ]);
         let mut writer = store
             .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
@@ -5806,6 +7877,193 @@ mod tests {
         ))
     }
 
+    #[test]
+    fn test_posting_block_size_schema_metadata() {
+        assert_eq!(parse_posting_block_size(&HashMap::new()).unwrap(), 128);
+
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "512".to_owned())]);
+        let err = parse_posting_block_size(&metadata).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
+
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "129".to_owned())]);
+        let err = parse_posting_block_size(&metadata).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
+    }
+
+    #[test]
+    fn test_num_tokens_only_reuses_sliced_arrow_storage() {
+        let docs = {
+            let source = UInt32Array::from(vec![999, 7, 16, 1024, 888]);
+            let sliced = source.slice(1, 3);
+            let mut docs = DocSet::from_num_tokens_only(&sliced);
+
+            let NumTokens::Shared(values) = &docs.num_tokens else {
+                panic!("num-tokens-only DocSet must retain shared Arrow storage");
+            };
+            assert!(values.ptr_eq(sliced.values()));
+            assert_eq!(values.as_ref(), &[7, 16, 1024]);
+            assert_eq!(docs.total_tokens_num(), 1047);
+            docs.set_quantized_scoring(true);
+            assert_eq!(docs.scoring_norms().unwrap().len(), 3);
+            assert_eq!(
+                docs.scoring_num_tokens(0),
+                dequantize_doc_length(quantize_doc_length(7))
+            );
+            assert_eq!(
+                docs.scoring_num_tokens(2),
+                dequantize_doc_length(quantize_doc_length(1024))
+            );
+            docs
+        };
+
+        assert_eq!(docs.len(), 3);
+        assert_eq!(docs.num_tokens(0), 7);
+        assert_eq!(docs.num_tokens(2), 1024);
+    }
+
+    #[test]
+    fn test_cached_num_tokens_uses_supplied_total_and_full_stays_owned() {
+        const CACHED_TOTAL_MARKER: u64 = 123_456;
+
+        let num_tokens = UInt32Array::from(vec![3, 5, 8]);
+        let docs = DocSet::from_cached_num_tokens(&num_tokens, CACHED_TOTAL_MARKER);
+        assert_eq!(docs.total_tokens_num(), CACHED_TOTAL_MARKER);
+        assert!(matches!(&docs.num_tokens, NumTokens::Shared(_)));
+
+        let row_ids = UInt64Array::from(vec![10, 20, 30]);
+        let full = DocSet::from_columns(&row_ids, &num_tokens, false, None).unwrap();
+        assert!(matches!(&full.num_tokens, NumTokens::Owned(_)));
+        assert_eq!(full.total_tokens_num(), 16);
+        assert_eq!(full.row_id(1), 20);
+    }
+
+    #[test]
+    fn test_posting_builder_writes_impacts_for_supported_block_sizes() {
+        for block_size in [128, 256] {
+            let format_version = default_fts_format_version_for_block_size(block_size).unwrap();
+            let num_docs = block_size * 33 + 1;
+            let mut docs = DocSet::default();
+            let mut posting = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                false,
+                format_version.posting_tail_codec(),
+                block_size,
+            );
+            for doc_id in 0..num_docs {
+                docs.append(doc_id as u64, (doc_id % 5 + 1) as u32);
+                posting.add(
+                    doc_id as u32,
+                    PositionRecorder::Count((doc_id % 3 + 1) as u32),
+                );
+            }
+            let schema =
+                inverted_list_schema_for_version_with_block_size(false, format_version, block_size);
+            let batch = posting.to_batch_with_docs(&docs, schema).unwrap();
+            assert!(batch.column_by_name(IMPACT_COL).is_some());
+            let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+            let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+            let posting = PostingList::from_batch(&batch, Some(max_score), Some(length)).unwrap();
+            let PostingList::Compressed(posting) = posting else {
+                panic!("expected compressed posting list");
+            };
+            let impacts = posting.impacts.expect("posting should include impacts");
+            assert_eq!(impacts.level0_len(), posting.blocks.len());
+            assert_eq!(impacts.level1_len(), posting.blocks.len().div_ceil(32));
+            assert_eq!(
+                impacts.entries().len(),
+                impacts.level0_len() + impacts.level1_len()
+            );
+        }
+    }
+
+    #[test]
+    fn test_posting_builder_without_impact_column_roundtrips_without_impacts() {
+        let mut posting = PostingListBuilder::new(false);
+        for doc_id in 0..BLOCK_SIZE + 3 {
+            posting.add(doc_id as u32, PositionRecorder::Count(1));
+        }
+        let batch = posting.to_batch(vec![1.0, 1.0]).unwrap();
+        assert!(batch.column_by_name(IMPACT_COL).is_none());
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some((BLOCK_SIZE + 3) as u32)).unwrap();
+        assert!(!posting.has_impacts());
+    }
+
+    #[tokio::test]
+    async fn test_build_search_uses_configured_posting_block_size() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let format_version = params.resolved_format_version();
+        let block_size = params.posting_block_size();
+        let num_docs = block_size + 7;
+
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+            block_size,
+        );
+        builder.tokens.add("needle".to_owned());
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            block_size,
+        );
+        for doc_id in 0..num_docs {
+            posting_list.add(doc_id as u32, PositionRecorder::Count(1));
+            builder.docs.append(1_000 + doc_id as u64, 1);
+        }
+        builder.posting_lists.push(posting_list);
+        builder.write(store.as_ref()).await.unwrap();
+        write_test_metadata(&store, vec![0], params).await;
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        assert_eq!(index.partitions[0].inverted_list.block_size(), block_size);
+
+        let posting = index.partitions[0]
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        let PostingList::Compressed(posting) = posting else {
+            panic!("expected compressed posting list");
+        };
+        assert_eq!(posting.block_size, block_size);
+        assert_eq!(posting.blocks.len(), num_docs.div_ceil(block_size));
+        let impacts = posting
+            .impacts
+            .as_ref()
+            .expect("newly written posting list should include impacts");
+        assert_eq!(impacts.level0_len(), posting.blocks.len());
+        assert_eq!(impacts.level1_len(), posting.blocks.len().div_ceil(32));
+        assert_eq!(
+            impacts.entries().len(),
+            impacts.level0_len() + impacts.level1_len()
+        );
+
+        let tokens = Arc::new(Tokens::new(vec!["needle".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+        let (row_ids, scores) = index
+            .bm25_search(tokens, params, Operator::Or, prefilter, metrics, None)
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids.len(), 10);
+        assert_eq!(scores.len(), 10);
+        assert!(row_ids.iter().all(|row_id| *row_id >= 1_000));
+    }
+
     #[tokio::test]
     async fn test_posting_builder_remap() {
         let posting_tail_codec = PostingTailCodec::Fixed32;
@@ -6045,6 +8303,20 @@ mod tests {
             resolve_fts_format_version(Some("2")).unwrap(),
             InvertedListFormatVersion::V2
         );
+        assert_eq!(
+            resolve_fts_format_version(Some("3")).unwrap(),
+            InvertedListFormatVersion::V3
+        );
+        assert!(resolve_fts_format_version(Some("4")).is_err());
+    }
+
+    #[test]
+    fn test_block_size_256_metadata_resolves_to_v3() {
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "256".to_owned())]);
+        assert_eq!(
+            parse_format_version_from_metadata(&metadata).unwrap(),
+            InvertedListFormatVersion::V3
+        );
     }
 
     #[test]
@@ -6468,37 +8740,534 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn test_modern_prewarm_shrinks_cached_posting_buffers() {
-        let tmpdir = TempObjDir::default();
-        let store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            tmpdir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
+    /// Counts docs-file ROW_ID reads so the test below can assert the
+    /// caching contract of deferred row_id resolution: one full-column read
+    /// per resolving partition on first use, zero scattered single-row reads,
+    /// and zero additional reads on subsequent queries.
+    #[derive(Debug, Default)]
+    struct DocsRowIdReadCounter {
+        full_column_reads: std::sync::atomic::AtomicUsize,
+        scattered_reads: std::sync::atomic::AtomicUsize,
+    }
 
-        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.tokens.add("alpha".to_owned());
-        builder.tokens.add("beta".to_owned());
-        builder.posting_lists.push(PostingListBuilder::new(false));
-        builder.posting_lists.push(PostingListBuilder::new(false));
-        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
-        builder.posting_lists[0].add(1, PositionRecorder::Count(2));
-        builder.posting_lists[1].add(2, PositionRecorder::Count(3));
-        builder.posting_lists[1].add(3, PositionRecorder::Count(4));
-        builder.docs.append(100, 1);
-        builder.docs.append(101, 2);
-        builder.docs.append(102, 3);
-        builder.docs.append(103, 4);
-        builder.write(store.as_ref()).await.unwrap();
+    #[cfg_attr(coverage, coverage(off))]
+    impl DocsRowIdReadCounter {
+        fn full_column_reads(&self) -> usize {
+            self.full_column_reads
+                .load(std::sync::atomic::Ordering::Relaxed)
+        }
+        fn scattered_reads(&self) -> usize {
+            self.scattered_reads
+                .load(std::sync::atomic::Ordering::Relaxed)
+        }
+    }
 
-        let metadata = std::collections::HashMap::from_iter(vec![
-            (
-                "partitions".to_owned(),
-                serde_json::to_string(&vec![0u64]).unwrap(),
-            ),
-            (
-                "params".to_owned(),
+    struct DocsRowIdCountingReader {
+        inner: Arc<dyn IndexReader>,
+        counter: Arc<DocsRowIdReadCounter>,
+    }
+
+    #[cfg_attr(coverage, coverage(off))]
+    #[async_trait]
+    impl IndexReader for DocsRowIdCountingReader {
+        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
+            self.inner.read_record_batch(n, batch_size).await
+        }
+        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
+            self.inner.read_global_buffer(index).await
+        }
+        async fn read_range(
+            &self,
+            range: std::ops::Range<usize>,
+            projection: Option<&[&str]>,
+        ) -> Result<RecordBatch> {
+            if projection
+                .map(|cols| cols.contains(&lance_core::ROW_ID))
+                .unwrap_or(false)
+            {
+                self.counter
+                    .full_column_reads
+                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            }
+            self.inner.read_range(range, projection).await
+        }
+        async fn read_ranges(
+            &self,
+            ranges: &[std::ops::Range<usize>],
+            projection: Option<&[&str]>,
+        ) -> Result<RecordBatch> {
+            if projection
+                .map(|cols| cols.contains(&lance_core::ROW_ID))
+                .unwrap_or(false)
+            {
+                self.counter
+                    .scattered_reads
+                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            }
+            self.inner.read_ranges(ranges, projection).await
+        }
+        async fn num_batches(&self, batch_size: u64) -> u32 {
+            self.inner.num_batches(batch_size).await
+        }
+        fn num_rows(&self) -> usize {
+            self.inner.num_rows()
+        }
+        fn schema(&self) -> &lance_core::datatypes::Schema {
+            self.inner.schema()
+        }
+    }
+
+    #[derive(Debug)]
+    struct DocsRowIdCountingStore {
+        inner: Arc<dyn IndexStore>,
+        counter: Arc<DocsRowIdReadCounter>,
+    }
+
+    #[cfg_attr(coverage, coverage(off))]
+    impl DeepSizeOf for DocsRowIdCountingStore {
+        fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+            self.inner.deep_size_of_children(context)
+        }
+    }
+
+    #[cfg_attr(coverage, coverage(off))]
+    #[async_trait]
+    impl IndexStore for DocsRowIdCountingStore {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.clone(),
+                counter: self.counter.clone(),
+            })
+        }
+        fn io_parallelism(&self) -> usize {
+            self.inner.io_parallelism()
+        }
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.with_io_priority(io_priority),
+                counter: self.counter.clone(),
+            })
+        }
+        async fn new_index_file(
+            &self,
+            name: &str,
+            schema: Arc<arrow_schema::Schema>,
+        ) -> Result<Box<dyn crate::scalar::IndexWriter>> {
+            self.inner.new_index_file(name, schema).await
+        }
+        async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
+            let reader = self.inner.open_index_file(name).await?;
+            if name.ends_with(DOCS_FILE) {
+                Ok(Arc::new(DocsRowIdCountingReader {
+                    inner: reader,
+                    counter: self.counter.clone(),
+                }))
+            } else {
+                Ok(reader)
+            }
+        }
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner.copy_index_file(name, dest_store).await
+        }
+        async fn copy_index_file_to(
+            &self,
+            name: &str,
+            new_name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner
+                .copy_index_file_to(name, new_name, dest_store)
+                .await
+        }
+        async fn rename_index_file(
+            &self,
+            name: &str,
+            new_name: &str,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner.rename_index_file(name, new_name).await
+        }
+        async fn delete_index_file(&self, name: &str) -> Result<()> {
+            self.inner.delete_index_file(name).await
+        }
+        async fn list_files_with_sizes(&self) -> Result<Vec<crate::scalar::IndexFile>> {
+            self.inner.list_files_with_sizes().await
+        }
+    }
+
+    /// Matching-partition count used by `write_many_partition_index`.
+    const MANY_PARTITIONS: u64 = 40;
+
+    /// Writes `MANY_PARTITIONS` partitions whose only token is "pipeline"
+    /// (1-3 docs each), one extra partition whose only token does not match,
+    /// and the metadata file. Returns every matching doc's row_id.
+    async fn write_many_partition_index(store: &LanceIndexStore) -> Vec<u64> {
+        let mut expected_row_ids: Vec<u64> = Vec::with_capacity((MANY_PARTITIONS * 3) as usize);
+        for pid in 0..MANY_PARTITIONS {
+            let mut builder = InnerBuilder::new(pid, false, TokenSetFormat::default());
+            builder.tokens.add("pipeline".to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+            let ndocs = (pid % 3) + 1;
+            for d in 0..ndocs {
+                builder.posting_lists[0].add(d as u32, PositionRecorder::Count(1));
+                let row_id = pid * 1000 + d;
+                builder.docs.append(row_id, 1);
+                expected_row_ids.push(row_id);
+            }
+            builder.write(store).await.unwrap();
+        }
+        // A partition whose only token does NOT match the query.
+        let mut empty_builder =
+            InnerBuilder::new(MANY_PARTITIONS, false, TokenSetFormat::default());
+        empty_builder.tokens.add("unrelated".to_owned());
+        empty_builder
+            .posting_lists
+            .push(PostingListBuilder::new(false));
+        empty_builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        empty_builder.docs.append(999_999, 1);
+        empty_builder.write(store).await.unwrap();
+
+        let all_partitions: Vec<u64> = (0..=MANY_PARTITIONS).collect();
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&all_partitions).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+        expected_row_ids
+    }
+
+    #[tokio::test]
+    async fn test_bm25_search_many_partitions_resolves_exact_row_ids() {
+        // Regression test for deferred row_id resolution: with many partitions
+        // (including one whose only token does not match the query), a search
+        // must resolve every candidate's row_id exactly once and correctly.
+        // Also asserts the caching contract: each resolving partition reads the
+        // ROW_ID column exactly once (one full-column read, no scattered
+        // single-row reads), and subsequent queries perform no further reads.
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected_row_ids = write_many_partition_index(store.as_ref()).await;
+
+        // Load through a store that counts docs-file ROW_ID reads.
+        let counter = Arc::new(DocsRowIdReadCounter::default());
+        let inner_store: Arc<dyn IndexStore> = store.clone();
+        let counting_store: Arc<dyn IndexStore> = Arc::new(DocsRowIdCountingStore {
+            inner: inner_store,
+            counter: counter.clone(),
+        });
+        let cache = Arc::new(LanceCache::with_capacity(64 * 1024 * 1024));
+        let index = InvertedIndex::load(counting_store, None, cache.as_ref())
+            .await
+            .unwrap();
+        assert_eq!(index.partitions.len(), 41);
+        assert_eq!(counter.full_column_reads(), 0);
+        assert_eq!(counter.scattered_reads(), 0);
+
+        // Every matching doc must come back exactly once with a correct row_id.
+        let tokens = Arc::new(Tokens::new(vec!["pipeline".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1000)));
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens.clone(),
+                params,
+                Operator::Or,
+                prefilter.clone(),
+                metrics.clone(),
+                None,
+            )
+            .await
+            .unwrap();
+        let mut got = row_ids.clone();
+        got.sort_unstable();
+        let mut want = expected_row_ids.clone();
+        want.sort_unstable();
+        assert_eq!(got, want, "every matching doc resolved exactly once");
+        assert_eq!(scores.len(), row_ids.len());
+        assert!(scores.iter().all(|s| *s > 0.0));
+
+        // Caching contract: one full-column ROW_ID read per resolving
+        // partition, no scattered single-row reads. The non-matching
+        // partition must not read its ROW_ID column at all.
+        assert_eq!(
+            counter.full_column_reads(),
+            MANY_PARTITIONS as usize,
+            "each resolving partition reads the ROW_ID column exactly once"
+        );
+        assert_eq!(
+            counter.scattered_reads(),
+            0,
+            "resolution must not issue scattered single-row reads"
+        );
+
+        // Top-k smaller than the total: exactly k results, and the cached
+        // columns serve resolution with no further docs-file reads.
+        let params_k = Arc::new(FtsSearchParams::new().with_limit(Some(7)));
+        let (row_ids_k, scores_k) = index
+            .bm25_search(tokens, params_k, Operator::Or, prefilter, metrics, None)
+            .await
+            .unwrap();
+        assert_eq!(row_ids_k.len(), 7);
+        assert_eq!(scores_k.len(), 7);
+        assert!(row_ids_k.iter().all(|id| expected_row_ids.contains(id)));
+        assert_eq!(
+            counter.full_column_reads(),
+            MANY_PARTITIONS as usize,
+            "subsequent queries must not re-read the ROW_ID column"
+        );
+        assert_eq!(counter.scattered_reads(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_bm25_search_resolves_only_topk_survivors_and_accounts_cache() {
+        // Late resolution: row_ids are resolved only for candidates that
+        // survive the global top-k, so with k much smaller than the partition
+        // count, only the partitions holding survivors load their ROW_ID
+        // column. Each loaded column must be visible to the index cache as
+        // its own weighed entry.
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected_row_ids = write_many_partition_index(store.as_ref()).await;
+
+        let counter = Arc::new(DocsRowIdReadCounter::default());
+        let inner_store: Arc<dyn IndexStore> = store.clone();
+        let counting_store: Arc<dyn IndexStore> = Arc::new(DocsRowIdCountingStore {
+            inner: inner_store,
+            counter: counter.clone(),
+        });
+        let cache = Arc::new(LanceCache::with_capacity(64 * 1024 * 1024));
+        let index = InvertedIndex::load(counting_store, None, cache.as_ref())
+            .await
+            .unwrap();
+
+        // k=3 with 40 matching partitions: survivors span at most 3
+        // partitions, so at most 3 ROW_ID columns are read even though every
+        // matching partition produced candidates.
+        let tokens = Arc::new(Tokens::new(vec!["pipeline".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(3)));
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(row_ids.len(), 3);
+        assert_eq!(scores.len(), 3);
+        assert!(row_ids.iter().all(|id| expected_row_ids.contains(id)));
+        let reads = counter.full_column_reads();
+        assert!(
+            (1..=3).contains(&reads),
+            "only partitions with surviving candidates load their ROW_ID column, got {reads}"
+        );
+        assert_eq!(counter.scattered_reads(), 0);
+
+        // Accounting: every loaded column is present in the index cache as a
+        // DocRowIds entry (weighed at insert, evictable under pressure).
+        // Partition caches are scoped with a `part-{id}` prefix on load.
+        let mut cached_entries = 0;
+        for partition_id in 0..=MANY_PARTITIONS {
+            if cache
+                .with_key_prefix(format!("part-{partition_id}").as_str())
+                .get_with_key(&DocRowIdsKey { partition_id })
+                .await
+                .is_some()
+            {
+                cached_entries += 1;
+            }
+        }
+        assert_eq!(
+            cached_entries, reads,
+            "each loaded ROW_ID column is its own index-cache entry"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_prewarm_fills_row_ids_cache_entry_without_a_second_copy() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected_row_ids = write_many_partition_index(store.as_ref()).await;
+
+        let counter = Arc::new(DocsRowIdReadCounter::default());
+        let inner_store: Arc<dyn IndexStore> = store.clone();
+        let counting_store: Arc<dyn IndexStore> = Arc::new(DocsRowIdCountingStore {
+            inner: inner_store,
+            counter: counter.clone(),
+        });
+        let cache = Arc::new(LanceCache::with_capacity(64 * 1024 * 1024));
+        let index = InvertedIndex::load(counting_store, None, cache.as_ref())
+            .await
+            .unwrap();
+
+        index
+            .prewarm_with_options(&FtsPrewarmOptions::default())
+            .await
+            .unwrap();
+        let total_partitions = (MANY_PARTITIONS + 1) as usize;
+        assert_eq!(counter.full_column_reads(), total_partitions);
+        assert_eq!(counter.scattered_reads(), 0);
+
+        for partition_id in 0..=MANY_PARTITIONS {
+            assert!(
+                cache
+                    .with_key_prefix(format!("part-{partition_id}").as_str())
+                    .get_with_key(&DocRowIdsKey { partition_id })
+                    .await
+                    .is_some(),
+                "prewarm must fill the DocRowIds entry for partition {partition_id}"
+            );
+        }
+        for part in &index.partitions {
+            let resident = part.docs.ensure_loaded().await.unwrap();
+            assert!(!resident.has_row_ids());
+            assert!(resident.supports_reverse_lookup());
+        }
+
+        let tokens = Arc::new(Tokens::new(vec!["pipeline".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1000)));
+        let (row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        let mut got = row_ids;
+        got.sort_unstable();
+        let mut want = expected_row_ids;
+        want.sort_unstable();
+        assert_eq!(got, want);
+        assert_eq!(counter.full_column_reads(), total_partitions);
+        assert_eq!(counter.scattered_reads(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_row_ids_resolution_reloads_after_eviction() {
+        // no_cache retains nothing — the always-evicted worst case: every
+        // query must reload the column and stay correct.
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let expected_row_ids = write_many_partition_index(store.as_ref()).await;
+
+        let counter = Arc::new(DocsRowIdReadCounter::default());
+        let inner_store: Arc<dyn IndexStore> = store.clone();
+        let counting_store: Arc<dyn IndexStore> = Arc::new(DocsRowIdCountingStore {
+            inner: inner_store,
+            counter: counter.clone(),
+        });
+        let cache = LanceCache::no_cache();
+        let index = InvertedIndex::load(counting_store, None, &cache)
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(vec!["pipeline".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1000)));
+        let mut want = expected_row_ids;
+        want.sort_unstable();
+        let mut reads_after_first = 0;
+        for round in 0..2 {
+            let (row_ids, _) = index
+                .bm25_search(
+                    tokens.clone(),
+                    params.clone(),
+                    Operator::Or,
+                    Arc::new(NoFilter),
+                    Arc::new(NoOpMetricsCollector),
+                    None,
+                )
+                .await
+                .unwrap();
+            let mut got = row_ids;
+            got.sort_unstable();
+            assert_eq!(got, want, "round {round} must stay correct");
+            if round == 0 {
+                reads_after_first = counter.full_column_reads();
+                assert!(reads_after_first > 0);
+            }
+        }
+        assert!(
+            counter.full_column_reads() > reads_after_first,
+            "with nothing retained, the next query reloads the column"
+        );
+        assert_eq!(counter.scattered_reads(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_modern_prewarm_packs_group_with_shared_posting_buffer() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.tokens.add("beta".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(1, PositionRecorder::Count(2));
+        builder.posting_lists[1].add(2, PositionRecorder::Count(3));
+        builder.posting_lists[1].add(3, PositionRecorder::Count(4));
+        builder.docs.append(100, 1);
+        builder.docs.append(101, 2);
+        builder.docs.append(102, 3);
+        builder.docs.append(103, 4);
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
                 serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
             ),
             (
@@ -6521,6 +9290,10 @@ mod tests {
             !inverted_list.is_legacy_layout(),
             "test should use modern posting layout"
         );
+        assert!(
+            inverted_list.has_impacts,
+            "modern posting fixture should include impact skip data"
+        );
 
         inverted_list.prewarm_posting_lists(false, 2).await.unwrap();
 
@@ -6529,108 +9302,215 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
 
-        let PostingList::Compressed(alpha) = group.get(0).unwrap() else {
+        assert!(
+            group.is_packed(),
+            "no-position prewarm should pack v2 groups"
+        );
+        assert!(
+            group.needs_external_metadata(),
+            "prewarmed packed groups must not duplicate reader score/length metadata"
+        );
+        let (alpha_score, alpha_len) = inverted_list.bulk_metadata_for_token(0);
+        let PostingList::Compressed(alpha) = group
+            .posting_list(0, alpha_score, alpha_len)
+            .unwrap()
+            .unwrap()
+        else {
             panic!("expected compressed posting list for token 0");
         };
-        let PostingList::Compressed(beta) = group.get(1).unwrap() else {
+        let PostingList::Compressed(alpha_again) = group
+            .posting_list(0, alpha_score, alpha_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list for repeated token 0 access");
+        };
+        let (beta_score, beta_len) = inverted_list.bulk_metadata_for_token(1);
+        let PostingList::Compressed(beta) = group
+            .posting_list(1, beta_score, beta_len)
+            .unwrap()
+            .unwrap()
+        else {
             panic!("expected compressed posting list for token 1");
         };
 
-        assert_ne!(
+        assert!(
+            alpha.impacts.is_some() && beta.impacts.is_some(),
+            "packed prewarm must preserve impact skip data"
+        );
+        assert!(
+            alpha
+                .impacts
+                .as_ref()
+                .unwrap()
+                .shares_derived_state_with(alpha_again.impacts.as_ref().unwrap()),
+            "repeated packed slot access must share decoded impact state"
+        );
+        assert!(
+            alpha.shares_first_docs_with(&alpha_again),
+            "repeated packed slot access must share decoded block heads"
+        );
+        assert_eq!(
+            alpha.block_first_docs().as_ptr(),
+            alpha_again.block_first_docs().as_ptr(),
+            "packed block heads should be decoded only once per slot"
+        );
+        assert_eq!(
             alpha.blocks.values().as_ptr(),
             beta.blocks.values().as_ptr(),
-            "prewarm should not leave cached posting lists sharing the same values buffer"
+            "packed posting views should share the group's values buffer"
         );
     }
 
-    #[test]
-    fn test_group_aligned_chunk_end_boundary_cases() {
-        let starts = [0, 3, 7, 10];
-        let token_count = 13;
+    #[tokio::test]
+    async fn test_packed_prewarm_groups_do_not_retain_the_full_chunk() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token_id in 0..4u32 {
+            builder.tokens.add(format!("t{token_id}"));
+            let mut posting = PostingListBuilder::new(false);
+            posting.add(token_id, PositionRecorder::Count(1));
+            builder.posting_lists.push(posting);
+            builder.docs.append(1000 + token_id as u64, 1);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+
+        let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
+        let cache = LanceCache::with_capacity(1 << 20);
+        let mut posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
+        posting_reader.grouping = PostingGrouping::SyntheticFixed { group_size: 2 };
 
         assert_eq!(
-            group_aligned_chunk_end(&starts, token_count, 0, 5),
-            3,
-            "chunk should snap back to the largest group boundary that fits"
+            posting_reader
+                .prewarm_posting_lists_chunked(false, Some(4), 1)
+                .await
+                .unwrap(),
+            1,
+            "the test must read both groups in one prewarm chunk"
         );
+
+        let first_group = posting_reader
+            .index_cache
+            .get_with_key(&posting_list_group_cache_key(
+                0,
+                2,
+                posting_reader.has_impacts,
+            ))
+            .await
+            .unwrap();
+        let second_group = posting_reader
+            .index_cache
+            .get_with_key(&posting_list_group_cache_key(
+                2,
+                4,
+                posting_reader.has_impacts,
+            ))
+            .await
+            .unwrap();
+        let (first_score, first_len) = posting_reader.bulk_metadata_for_token(0);
+        let PostingList::Compressed(first) = first_group
+            .posting_list(0, first_score, first_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in first group");
+        };
+        let (neighbor_score, neighbor_len) = posting_reader.bulk_metadata_for_token(1);
+        let PostingList::Compressed(first_neighbor) = first_group
+            .posting_list(1, neighbor_score, neighbor_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in first group");
+        };
+        let (second_score, second_len) = posting_reader.bulk_metadata_for_token(2);
+        let PostingList::Compressed(second) = second_group
+            .posting_list(0, second_score, second_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in second group");
+        };
+
         assert_eq!(
-            group_aligned_chunk_end(&starts, token_count, 3, 6),
-            7,
-            "oversized groups should run as one chunk"
+            first.blocks.values().as_ptr(),
+            first_neighbor.blocks.values().as_ptr(),
+            "postings in one group should share the group's values buffer"
         );
-        assert_eq!(
-            group_aligned_chunk_end(&starts, token_count, 7, 10),
-            10,
-            "an exact next group boundary should be selected"
+        assert_ne!(
+            first.blocks.values().as_ptr(),
+            second.blocks.values().as_ptr(),
+            "each group must own a compact buffer instead of retaining the full chunk"
         );
+    }
+
+    #[test]
+    fn test_prewarm_chunk_ranges_preserve_group_boundaries() {
+        let grouping = PostingGrouping::SyntheticFixed { group_size: 4 };
         assert_eq!(
-            group_aligned_chunk_end(&starts, token_count, 10, 12),
-            13,
-            "the last group should extend to token_count"
+            prewarm_chunk_ranges(&grouping, 13, 5),
+            vec![(0, 4), (4, 8), (8, 13)],
+            "grouped chunks may contain multiple groups but must never split one"
         );
         assert_eq!(
-            group_aligned_chunk_end(&starts, token_count, 7, 13),
-            13,
-            "token_count should act as the final boundary"
+            prewarm_chunk_ranges(&PostingGrouping::None, 13, 5),
+            vec![(0, 5), (5, 10), (10, 13)],
+            "ungrouped chunk ranges should use plain token ranges"
         );
     }
 
     #[test]
-    fn test_group_start_indices_for_chunk_boundary_cases() {
-        let starts = [0, 3, 7, 10];
-        let token_count = 13;
-        let ranges_for_chunk = |tok_start, tok_end| {
-            group_start_indices_for_chunk(&starts, tok_start, tok_end)
-                .map(|group_idx| group_range_for_start_index(&starts, token_count, group_idx))
-                .collect::<Vec<_>>()
-        };
-
+    fn test_synthetic_grouping_preserves_fixed_boundaries() {
+        let grouping = PostingGrouping::SyntheticFixed { group_size: 4 };
         assert_eq!(
-            ranges_for_chunk(0, 7),
-            vec![(0, 3), (3, 7)],
-            "publish should include only groups that start in the chunk"
+            grouping.range_for_token(5, 10),
+            Some((4, 8)),
+            "synthetic token groups should be fixed-size ranges"
         );
         assert_eq!(
-            ranges_for_chunk(7, 13),
-            vec![(7, 10), (10, 13)],
-            "publish should include the final group ending at token_count"
+            grouping.range_for_token(9, 10),
+            Some((8, 10)),
+            "the final synthetic group should end at token_count"
         );
         assert_eq!(
-            ranges_for_chunk(3, 10),
-            vec![(3, 7), (7, 10)],
-            "publish selection should work for an interior chunk"
+            prewarm_chunk_ranges(&grouping, 10, 6),
+            vec![(0, 4), (4, 10)],
+            "prewarm chunks may contain multiple synthetic groups but must not split one"
         );
-    }
-
-    #[test]
-    fn test_prewarm_chunk_ranges_preserve_group_boundaries() {
-        let starts = [0, 3, 7, 10];
         assert_eq!(
-            prewarm_chunk_ranges(Some(&starts), 13, 5),
-            vec![(0, 3), (3, 7), (7, 10), (10, 13)],
-            "grouped chunk ranges must never split a posting cache group"
-        );
-        assert_eq!(
-            prewarm_chunk_ranges(None, 13, 5),
-            vec![(0, 5), (5, 10), (10, 13)],
-            "ungrouped chunk ranges should use plain token ranges"
+            grouping.ranges_for_chunk(4, 10, 10),
+            vec![(4, 8), (8, 10)],
+            "publish selection should enumerate synthetic groups in a chunk"
         );
     }
 
     /// Prewarming a large partition in multiple chunks must end up holding exactly the
     /// same per-token posting lists (doc ids and frequencies) as the whole-file path.
     /// Parametrized over layout: the legacy-v1 chunk path rebases global offsets to
-    /// chunk-local rows, which the v2 one-row-per-token path never exercises.
+    /// chunk-local rows, while the modern one-row-per-token path covers both
+    /// legacy-sized v2 and 256-doc v3 posting blocks.
     #[rstest::rstest]
-    #[case::v1(InvertedListFormatVersion::V1)]
-    #[case::v2(InvertedListFormatVersion::V2)]
+    #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)]
+    #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)]
+    #[case::v3(InvertedListFormatVersion::V3, 256)]
     #[tokio::test]
     async fn test_prewarm_streams_in_chunks_preserves_content(
         #[case] format_version: InvertedListFormatVersion,
+        #[case] block_size: usize,
     ) {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
@@ -6639,30 +9519,28 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // One partition with many tokens (so it spans many chunks) and several
-        // docs per token (so each token is more than one posting row).
-        const NUM_TOKENS: u32 = 20;
+        // One partition with enough tokens to span multiple runtime synthetic
+        // groups and several docs per token.
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         const DOCS_PER_TOKEN: u32 = 3;
         let posting_tail_codec = format_version.posting_tail_codec();
-        let mut builder = InnerBuilder::new_with_format_version(
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
             0,
             false,
             TokenSetFormat::default(),
             format_version,
+            block_size,
         );
-        // Small groups so the partition spans several; chunks snap to whole groups,
-        // so several groups are needed to stream in more than one chunk.
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 4,
-        };
         // expected[token] = [(doc_id, frequency)] in stored (doc-id) order.
         let mut expected: Vec<Vec<(u32, u32)>> = Vec::new();
         let mut doc_id = 0u64;
-        for t in 0..NUM_TOKENS {
+        for t in 0..num_tokens {
             builder.tokens.add(format!("tok_{t:03}"));
-            let mut posting =
-                PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+            let mut posting = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                false,
+                posting_tail_codec,
+                block_size,
+            );
             let mut docs = Vec::new();
             for _ in 0..DOCS_PER_TOKEN {
                 posting.add(doc_id as u32, PositionRecorder::Count(1));
@@ -6675,15 +9553,15 @@ mod tests {
         }
         builder.write(store.as_ref()).await.unwrap();
 
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)
+            .unwrap();
         let metadata = std::collections::HashMap::from_iter(vec![
-            (
-                "partitions".to_owned(),
-                serde_json::to_string(&vec![0u64]).unwrap(),
-            ),
-            (
-                "params".to_owned(),
-                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
             ),
+            ("params".to_owned(), serde_json::to_string(&params).unwrap()),
             (
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 TokenSetFormat::default().to_string(),
@@ -6692,6 +9570,11 @@ mod tests {
                 POSTING_TAIL_CODEC_KEY.to_owned(),
                 posting_tail_codec.as_str().to_owned(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
         ]);
         let mut writer = store
             .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
@@ -6704,10 +9587,12 @@ mod tests {
             .await
             .unwrap();
         let inverted_list = &index.partitions[0].inverted_list;
-        assert_eq!(inverted_list.len(), NUM_TOKENS as usize);
+        assert_eq!(inverted_list.len(), num_tokens as usize);
+        assert_eq!(inverted_list.block_size(), block_size);
 
-        // Force a small chunk so the partition deterministically splits; with
-        // CHUNK_TOKENS < NUM_TOKENS each chunk is bounded below the whole partition.
+        // Force a small target chunk. Since CHUNK_TOKENS is below the runtime
+        // group size, synthetic group alignment should still split only at
+        // group boundaries.
         const CHUNK_TOKENS: usize = 6;
         let chunk_count = inverted_list
             .prewarm_posting_lists_chunked(false, Some(CHUNK_TOKENS), 2)
@@ -6722,9 +9607,34 @@ mod tests {
             "single partition must be streamed in more than one chunk, got {chunk_count}"
         );
 
+        if block_size == 256 {
+            let (start, end) = inverted_list.group_range_for_token(0).unwrap();
+            let group = inverted_list
+                .index_cache
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    inverted_list.has_impacts,
+                ))
+                .await
+                .expect("256-document blocks should populate the packed group cache");
+            assert!(group.is_packed());
+            let (max_score, length) = inverted_list.bulk_metadata_for_token(0);
+            let PostingList::Compressed(posting) =
+                group.posting_list(0, max_score, length).unwrap().unwrap()
+            else {
+                panic!("expected compressed posting list");
+            };
+            assert_eq!(posting.block_size, 256);
+            assert!(
+                posting.impacts.is_some(),
+                "packed prewarm must preserve impact skip data"
+            );
+        }
+
         // (2) Correctness: every token's posting list round-trips with exactly
         // the doc ids and frequencies of the whole-file path.
-        for token_id in 0..NUM_TOKENS {
+        for token_id in 0..num_tokens {
             let actual = inverted_list
                 .posting_list(token_id, false, &NoOpMetricsCollector)
                 .await
@@ -6753,7 +9663,7 @@ mod tests {
 
         let format_version = InvertedListFormatVersion::V2;
         let posting_tail_codec = format_version.posting_tail_codec();
-        const NUM_TOKENS: u32 = 16;
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         const DOCS_PER_TOKEN: u32 = 3;
         let mut builder = InnerBuilder::new_with_format_version(
             0,
@@ -6761,14 +9671,10 @@ mod tests {
             TokenSetFormat::default(),
             format_version,
         );
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 4,
-        };
         // expected[token] = [(doc_id, frequency, positions)].
         let mut expected: Vec<Vec<(u32, u32, Vec<u32>)>> = Vec::new();
         let mut doc_id = 0u64;
-        for t in 0..NUM_TOKENS {
+        for t in 0..num_tokens {
             builder.tokens.add(format!("tok_{t:03}"));
             let mut posting =
                 PostingListBuilder::new_with_posting_tail_codec(true, posting_tail_codec);
@@ -6836,17 +9742,29 @@ mod tests {
             "partition must be streamed in more than one chunk, got {chunk_count}"
         );
 
-        for token_id in 0..NUM_TOKENS {
+        for token_id in 0..num_tokens {
             // The prewarmed posting cache entry is positions-free.
             let (start, end) = inverted_list.group_range_for_token(token_id).unwrap();
             let group = inverted_list
                 .index_cache
-                .get_with_key(&PostingListGroupKey { start, end })
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    inverted_list.has_impacts,
+                ))
                 .await
                 .unwrap();
             let slot = (token_id - start) as usize;
             assert!(
-                !group.get(slot).unwrap().has_position(),
+                !group.is_packed(),
+                "with-position prewarm should retain the materialized fallback"
+            );
+            assert!(
+                !group
+                    .posting_list(slot, None, None)
+                    .unwrap()
+                    .unwrap()
+                    .has_position(),
                 "token {token_id} posting cache entry must be positions-free after prewarm"
             );
 
@@ -7175,6 +10093,80 @@ mod tests {
         assert_eq!(second, first);
     }
 
+    #[tokio::test]
+    async fn test_stats_then_num_tokens_view_reuses_shared_storage() {
+        let (index, _counter, _tmpdir) = load_counted_v2_index(100, LanceCache::no_cache()).await;
+        let partition = index.partitions[0].clone();
+
+        assert_eq!(index.aggregate_corpus_stats().await.unwrap(), (100, 100));
+        assert_eq!(partition.docs.total_tokens_cached(), Some(100));
+
+        let views =
+            futures::future::join_all((0..8).map(|_| partition.docs.ensure_num_tokens_loaded()))
+                .await
+                .into_iter()
+                .collect::<Result<Vec<_>>>()
+                .unwrap();
+        let first = &views[0];
+        assert!(views.iter().all(|view| Arc::ptr_eq(first, view)));
+        assert!(!first.has_row_ids());
+        assert!(matches!(&first.num_tokens, NumTokens::Shared(_)));
+        assert_eq!(first.total_tokens_num(), 100);
+
+        let all_rows = RowAddrMask::all_rows();
+        let wand_view = partition
+            .docs
+            .docs_for_wand(Operator::Or, &all_rows)
+            .await
+            .unwrap();
+        assert!(Arc::ptr_eq(first, &wand_view));
+
+        // Flat-shaped mask (OR + tiny allow-list) → resident set with `inv`.
+        let filtered = RowAddrMask::allow_nothing();
+        let resident = partition
+            .docs
+            .docs_for_wand(Operator::Or, &filtered)
+            .await
+            .unwrap();
+        assert!(!resident.has_row_ids());
+        assert!(resident.supports_reverse_lookup());
+        assert!(matches!(&resident.num_tokens, NumTokens::Shared(_)));
+        assert_eq!(resident.total_tokens_num(), 100);
+
+        // Masked non-flat walk (AND is never flat) → per-query view.
+        let masked_view = partition
+            .docs
+            .docs_for_wand(Operator::And, &filtered)
+            .await
+            .unwrap();
+        assert!(masked_view.has_row_ids());
+        assert!(matches!(&masked_view.row_ids, RowIds::Shared(_)));
+        assert_eq!(masked_view.total_tokens_num(), 100);
+
+        assert_eq!(
+            partition.docs.resolve_row_ids(&[0, 99]).await.unwrap(),
+            [0, 99]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_total_and_num_tokens_view_initialization() {
+        let (index, _counter, _tmpdir) = load_counted_v2_index(100, LanceCache::no_cache()).await;
+        let docs = index.partitions[0].docs.clone();
+
+        let totals = futures::future::join_all((0..8).map(|_| docs.total_tokens_num()));
+        let views = futures::future::join_all((0..8).map(|_| docs.ensure_num_tokens_loaded()));
+        let (totals, views) = tokio::join!(totals, views);
+
+        let totals = totals.into_iter().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(totals, vec![100; 8]);
+        let views = views.into_iter().collect::<Result<Vec<_>>>().unwrap();
+        let first = &views[0];
+        assert!(views.iter().all(|view| Arc::ptr_eq(first, view)));
+        assert!(matches!(&first.num_tokens, NumTokens::Shared(_)));
+        assert_eq!(docs.total_tokens_cached(), Some(100));
+    }
+
     #[tokio::test]
     async fn test_grouped_posting_lists_read_one_group_per_neighborhood() {
         // Cold-start scoring must not bulk-read the full `0..num_tokens`
@@ -7182,8 +10174,13 @@ mod tests {
         // K adjacent cold tokens shares a single group cache entry: one
         // read_range bounded by the group size, independent of the partition's
         // total token count.
-        let num_tokens = 500;
-        let queried_tokens: [u32; 4] = [0, 1, 2, 3];
+        let runtime_group_size = runtime_posting_group_tokens().max(1);
+        let queried_token_count = runtime_group_size.min(4);
+        let queried_tokens = (0..queried_token_count as u32).collect::<Vec<_>>();
+        let num_tokens = runtime_group_size
+            .saturating_mul(2)
+            .max(queried_token_count + 1)
+            .min(1024);
         let (index, counter, _tmpdir) =
             load_counted_v2_index(num_tokens, LanceCache::no_cache()).await;
         let inverted_list = index.partitions[0].inverted_list.clone();
@@ -7192,15 +10189,18 @@ mod tests {
             "this test only proves the lazy path for v2 indexes",
         );
         assert!(
-            inverted_list.group_starts.is_some(),
-            "freshly written v2 index should carry posting group offsets",
+            matches!(
+                &inverted_list.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "freshly written v2 index should use runtime synthetic groups",
         );
 
         // This fixture uses a no-op cache, so each call re-reads; that isolates
         // the per-query read shape. Each posting_list call reads exactly its
         // own group — bounded by the group size, never the full token table.
         let metrics = Arc::new(NoOpMetricsCollector);
-        for token_id in queried_tokens {
+        for &token_id in &queried_tokens {
             inverted_list
                 .posting_list(token_id, false, metrics.as_ref())
                 .await
@@ -7210,9 +10210,9 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group_len = (end - start) as usize;
         assert!(
-            (queried_tokens.len()..num_tokens).contains(&group_len),
-            "group [{start}, {end}) should cover the queried neighborhood but be \
-             far smaller than the {num_tokens}-token table",
+            (queried_tokens.len()..=num_tokens).contains(&group_len),
+            "group [{start}, {end}) should cover the queried neighborhood and \
+             stay bounded by the {num_tokens}-token table",
         );
         assert_eq!(
             counter.read_range_calls(),
@@ -7228,8 +10228,8 @@ mod tests {
     }
 
     /// Build a single-partition v2 index where every token's posting list spans
-    /// `docs_per_token` docs. Small `docs_per_token` yields tiny posting lists
-    /// that the writer packs densely into shared cache groups.
+    /// `docs_per_token` docs. Runtime grouping packs consecutive token rows
+    /// into shared cache groups.
     async fn load_v2_index_with_grouped_postings(
         num_tokens: usize,
         docs_per_token: usize,
@@ -7286,25 +10286,18 @@ mod tests {
         (index, cache)
     }
 
-    /// The read path decodes a posting-list group by slicing one buffer read for
-    /// the whole `[start, end)` row range, so every posting list in a cached
-    /// group shares a single `blocks` buffer. `DeepSizeOf` must count each
-    /// posting's slice of that buffer, not the whole buffer once per posting —
-    /// otherwise a group of N postings reports ~N times its real footprint.
-    #[rstest::rstest]
-    #[case::single_doc_terms(512, 1)]
-    #[case::small_terms(512, 4)]
-    #[case::medium_terms(256, 32)]
+    /// Packed groups charge their Arrow buffers and contiguous metadata once,
+    /// avoiding the per-member enum/array object graph of a materialized group.
     #[tokio::test]
-    async fn test_read_path_group_size_counts_slices_not_shared_buffer(
-        #[case] num_tokens: usize,
-        #[case] docs_per_token: usize,
-    ) {
-        let (index, _cache) = load_v2_index_with_grouped_postings(num_tokens, docs_per_token).await;
+    async fn test_packed_group_deep_size_is_smaller_than_materialized_graph() {
+        let (index, _cache) = load_v2_index_with_grouped_postings(512, 1).await;
         let inverted_list = index.partitions[0].inverted_list.clone();
         assert!(!inverted_list.is_legacy_layout(), "expected v2 layout");
         assert!(
-            inverted_list.group_starts.is_some(),
+            matches!(
+                &inverted_list.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
             "expected grouped posting lists"
         );
 
@@ -7316,22 +10309,31 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
+        assert!(group.is_packed(), "cold v2 group should use packed storage");
+        inverted_list.ensure_metadata_loaded().await.unwrap();
 
-        // Sum what counting the full backing buffer once per posting list would
-        // charge, and confirm the postings really do share a single buffer.
         let mut distinct_buffers = std::collections::HashSet::new();
-        let mut charged_if_counted_per_posting = 0usize;
-        for posting in &group.posting_lists {
+        let mut materialized = Vec::with_capacity(group.len());
+        for slot in 0..group.len() {
+            let (max_score, length) = inverted_list.bulk_metadata_for_token(start + slot as u32);
+            let posting = group
+                .posting_list(slot, max_score, length)
+                .unwrap()
+                .unwrap();
             let PostingList::Compressed(compressed) = posting else {
                 panic!("expected compressed posting lists");
             };
-            charged_if_counted_per_posting += compressed.blocks.get_buffer_memory_size();
             distinct_buffers.insert(compressed.blocks.values().as_ptr());
+            materialized.push(PostingList::Compressed(compressed));
         }
-        let posting_count = group.posting_lists.len();
+        let posting_count = materialized.len();
 
         assert!(
             posting_count > 1,
@@ -7342,13 +10344,12 @@ mod tests {
             1,
             "read-path postings in a group should share one backing buffer"
         );
-        // With slice-aware accounting the shared buffer is counted ~once, so the
-        // whole group costs far less than counting it once per posting list.
-        let reported = group.deep_size_of();
+        let packed_size = group.deep_size_of();
+        let materialized_size = PostingListGroup::new(materialized).deep_size_of();
         assert!(
-            reported < charged_if_counted_per_posting / 2,
-            "group deep_size_of {reported}B should not scale with the {posting_count}x-counted \
-             shared buffer ({charged_if_counted_per_posting}B)"
+            packed_size * 4 < materialized_size * 3,
+            "packed group deep_size_of {packed_size}B should be at least 25% smaller than the \
+             {materialized_size}B materialized graph for {posting_count} postings"
         );
     }
 
@@ -7403,6 +10404,8 @@ mod tests {
             1.0,
             SLICE_LEN as u32,
             PostingTailCodec::Fixed32,
+            LEGACY_BLOCK_SIZE,
+            None,
             None,
         );
 
@@ -7550,11 +10553,23 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
         assert!(
-            !group.get(0).unwrap().has_position(),
+            !group.is_packed(),
+            "with-position prewarm should retain the materialized fallback"
+        );
+        assert!(
+            !group
+                .posting_list(0, None, None)
+                .unwrap()
+                .unwrap()
+                .has_position(),
             "posting cache should remain positions-free after prewarm"
         );
 
@@ -7758,27 +10773,409 @@ mod tests {
         }
     }
 
-    async fn write_test_metadata(
-        store: &Arc<LanceIndexStore>,
-        partition_ids: Vec<u64>,
-        params: InvertedIndexParams,
-    ) {
-        let metadata = HashMap::from([
-            (
-                "partitions".to_owned(),
-                serde_json::to_string(&partition_ids).unwrap(),
-            ),
-            ("params".to_owned(), serde_json::to_string(&params).unwrap()),
-            (
-                TOKEN_SET_FORMAT_KEY.to_owned(),
-                TokenSetFormat::default().to_string(),
-            ),
-        ]);
-        let mut writer = store
-            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+    async fn write_test_metadata(
+        store: &Arc<LanceIndexStore>,
+        partition_ids: Vec<u64>,
+        params: InvertedIndexParams,
+    ) {
+        let format_version = params.resolved_format_version();
+        let metadata = HashMap::from([
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&partition_ids).unwrap(),
+            ),
+            ("params".to_owned(), serde_json::to_string(&params).unwrap()),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                format_version.posting_tail_codec().as_str().to_owned(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                params.posting_block_size().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+    }
+
+    async fn write_test_partition_with_optional_impacts(
+        store: &Arc<LanceIndexStore>,
+        partition_id: u64,
+        mut builder: InnerBuilder,
+        token_set_format: TokenSetFormat,
+        with_impacts: bool,
+    ) {
+        let format_version = InvertedListFormatVersion::V1;
+        let block_size = LEGACY_BLOCK_SIZE;
+        let docs = std::mem::take(&mut builder.docs);
+        let schema = inverted_list_schema_for_version_with_block_size_and_impacts(
+            false,
+            format_version,
+            block_size,
+            with_impacts,
+        );
+
+        let mut posting_writer = store
+            .new_index_file(&posting_file_path(partition_id), schema.clone())
+            .await
+            .unwrap();
+        for posting_list in std::mem::take(&mut builder.posting_lists) {
+            let batch = posting_list
+                .to_batch_with_docs(&docs, schema.clone())
+                .unwrap();
+            posting_writer.write_record_batch(batch).await.unwrap();
+        }
+        posting_writer.finish().await.unwrap();
+
+        let token_batch = std::mem::take(&mut builder.tokens)
+            .to_batch(token_set_format)
+            .unwrap();
+        let mut token_writer = store
+            .new_index_file(&token_file_path(partition_id), token_batch.schema())
+            .await
+            .unwrap();
+        token_writer.write_record_batch(token_batch).await.unwrap();
+        token_writer.finish().await.unwrap();
+
+        let doc_batch = docs.to_batch().unwrap();
+        let mut doc_writer = store
+            .new_index_file(&doc_file_path(partition_id), doc_batch.schema())
+            .await
+            .unwrap();
+        doc_writer.write_record_batch(doc_batch).await.unwrap();
+        doc_writer.finish().await.unwrap();
+    }
+
+    async fn load_global_scoring_test_index(
+        first_partition_has_impacts: bool,
+        second_partition_has_impacts: bool,
+    ) -> (TempObjDir, Arc<InvertedIndex>) {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let partition_specs = [
+            (0, 100, 5_000, 101..111, 5_000, first_partition_has_impacts),
+            (1, 200, 1_000, 201..301, 1, second_partition_has_impacts),
+        ];
+        for (
+            partition_id,
+            matching_row_id,
+            matching_doc_length,
+            other_row_ids,
+            other_doc_length,
+            with_impacts,
+        ) in partition_specs
+        {
+            let mut builder = InnerBuilder::new_with_format_version(
+                partition_id,
+                false,
+                TokenSetFormat::default(),
+                InvertedListFormatVersion::V1,
+            );
+            builder.tokens.add("alpha".to_owned());
+            builder
+                .posting_lists
+                .push(PostingListBuilder::new_with_posting_tail_codec(
+                    false,
+                    InvertedListFormatVersion::V1.posting_tail_codec(),
+                ));
+            builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+            builder.docs.append(matching_row_id, matching_doc_length);
+            for row_id in other_row_ids {
+                builder.docs.append(row_id, other_doc_length);
+            }
+            write_test_partition_with_optional_impacts(
+                &store,
+                partition_id,
+                builder,
+                TokenSetFormat::default(),
+                with_impacts,
+            )
+            .await;
+        }
+
+        write_test_metadata(&store, vec![0, 1], InvertedIndexParams::default()).await;
+        let cache = LanceCache::with_capacity(4096);
+        let index = InvertedIndex::load(store, None, &cache).await.unwrap();
+        (tmpdir, index)
+    }
+
+    async fn search_test_impact_partition(
+        partition: &InvertedPartition,
+        tokens: &Tokens,
+        params: &FtsSearchParams,
+        scorer: Arc<MemBM25Scorer>,
+        shared_threshold: Arc<AtomicU32>,
+    ) -> Vec<DocCandidate> {
+        let LoadedPostings {
+            postings,
+            grouped_expansions,
+            impact_safe,
+            exact_scoring_required,
+        } = partition
+            .load_posting_lists(
+                tokens,
+                params,
+                Operator::Or,
+                scorer.as_ref(),
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        assert!(impact_safe);
+        assert!(!exact_scoring_required);
+        assert!(grouped_expansions.is_empty());
+
+        let mask = NoFilter.mask();
+        let docs_for_wand = partition
+            .docs
+            .docs_for_wand(Operator::Or, mask.as_ref())
+            .await
+            .unwrap();
+        let mut candidates = partition
+            .bm25_search(
+                docs_for_wand.as_ref(),
+                params,
+                Operator::Or,
+                mask,
+                postings,
+                Some(scorer),
+                &NoOpMetricsCollector,
+                shared_threshold,
+            )
+            .unwrap();
+        resolve_deferred_candidates(&partition.docs, &mut candidates)
+            .await
+            .unwrap();
+        candidates
+    }
+
+    #[tokio::test]
+    async fn test_impact_partitions_share_global_threshold_without_pruning_winner() {
+        // Partition 0 wins under its local corpus statistics but loses under
+        // the global statistics. If its local score escapes into the shared
+        // floor, partition 1 will incorrectly prune the real global winner.
+        let (_tmpdir, index) = load_global_scoring_test_index(true, true).await;
+        let first_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 0)
+            .unwrap();
+        let second_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 1)
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(vec!["alpha".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let scorer = Arc::new(
+            index
+                .bm25_base_scorer(tokens.as_ref(), params.as_ref())
+                .await
+                .unwrap(),
+        );
+        first_partition
+            .inverted_list
+            .ensure_metadata_loaded()
+            .await
+            .unwrap();
+        second_partition
+            .inverted_list
+            .ensure_metadata_loaded()
+            .await
+            .unwrap();
+        let first_local_scorer = IndexBM25Scorer::new(std::iter::once(first_partition.as_ref()));
+        let second_local_scorer = IndexBM25Scorer::new(std::iter::once(second_partition.as_ref()));
+        let first_local_score =
+            first_local_scorer.query_weight("alpha") * first_local_scorer.doc_weight(1, 5_000);
+        let second_local_score =
+            second_local_scorer.query_weight("alpha") * second_local_scorer.doc_weight(1, 1_000);
+        assert!(first_local_score > second_local_score);
+        let shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+
+        // Search sequentially so partition 0 deterministically publishes its
+        // score before partition 1 evaluates its impact upper bound.
+        let first_candidates = search_test_impact_partition(
+            first_partition,
+            tokens.as_ref(),
+            params.as_ref(),
+            scorer.clone(),
+            shared_threshold.clone(),
+        )
+        .await;
+        assert_eq!(first_candidates.len(), 1);
+        assert!(matches!(
+            first_candidates[0].addr,
+            CandidateAddr::RowId(100)
+        ));
+        let first_score =
+            scorer.query_weight("alpha") * scorer.doc_weight(1, first_candidates[0].doc_length);
+        let published_threshold = f32::from_bits(shared_threshold.load(Ordering::Relaxed));
+        assert!(
+            (published_threshold - first_score).abs() < 1e-6,
+            "published threshold: {published_threshold}, expected global score: {first_score}"
+        );
+
+        let second_candidates = search_test_impact_partition(
+            second_partition,
+            tokens.as_ref(),
+            params.as_ref(),
+            scorer.clone(),
+            shared_threshold.clone(),
+        )
+        .await;
+        assert_eq!(second_candidates.len(), 1);
+        assert!(matches!(
+            second_candidates[0].addr,
+            CandidateAddr::RowId(200)
+        ));
+        let second_score =
+            scorer.query_weight("alpha") * scorer.doc_weight(1, second_candidates[0].doc_length);
+        assert!(
+            second_score > first_score,
+            "second score: {second_score}, first score: {first_score}"
+        );
+        assert!(
+            (f32::from_bits(shared_threshold.load(Ordering::Relaxed)) - second_score).abs() < 1e-6
+        );
+
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(row_ids, vec![200]);
+        assert_eq!(scores.len(), 1);
+        assert!((scores[0] - second_score).abs() < 1e-6);
+    }
+
+    #[tokio::test]
+    async fn test_mixed_impact_and_legacy_partitions_use_global_final_scores() {
+        let (_tmpdir, index) = load_global_scoring_test_index(true, false).await;
+
+        let impact_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 0)
+            .unwrap();
+        let legacy_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 1)
+            .unwrap();
+
+        let impact_posting = impact_partition
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(impact_posting.has_impacts());
+
+        let legacy_posting = legacy_partition
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(!legacy_posting.has_impacts());
+
+        let tokens = Arc::new(Tokens::new(vec!["alpha".to_string()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens.clone(),
+                params.clone(),
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids, vec![200]);
+        assert_eq!(row_ids.len(), scores.len());
+
+        let scorer = index
+            .bm25_base_scorer(tokens.as_ref(), params.as_ref())
+            .await
+            .unwrap();
+        let expected_score = scorer.query_weight("alpha") * scorer.doc_weight(1, 1_000);
+        assert!(
+            (scores[0] - expected_score).abs() < 1e-6,
+            "score: {}, expected: {}",
+            scores[0],
+            expected_score
+        );
+    }
+
+    #[tokio::test]
+    async fn test_two_legacy_partitions_keep_private_thresholds() {
+        // Both partitions are legacy (no impacts), so their BM25 statistics
+        // are partition-local scale. Partition 0's matching doc scores high
+        // under its own statistics while partition 1's matching doc (the true
+        // global winner, row 200) scores low under partition 1's local
+        // statistics. The chunked search runs both partitions sequentially in
+        // one chunk: if partition 0's local k-th score leaked into a shared
+        // threshold, partition 1 would prune row 200 before global rescoring
+        // and return the wrong winner.
+        let (_tmpdir, index) = load_global_scoring_test_index(false, false).await;
+        for partition in index.partitions.iter() {
+            let posting = partition
+                .inverted_list
+                .posting_list(0, false, &NoOpMetricsCollector)
+                .await
+                .unwrap();
+            assert!(!posting.has_impacts());
+        }
+
+        let tokens = Arc::new(Tokens::new(vec!["alpha".to_string()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens.clone(),
+                params.clone(),
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids, vec![200]);
+        assert_eq!(scores.len(), 1);
+        let scorer = index
+            .bm25_base_scorer(tokens.as_ref(), params.as_ref())
             .await
             .unwrap();
-        writer.finish_with_metadata(metadata).await.unwrap();
+        let expected_score = scorer.query_weight("alpha") * scorer.doc_weight(1, 1_000);
+        assert!(
+            (scores[0] - expected_score).abs() < 1e-6,
+            "score: {}, expected global score: {}",
+            scores[0],
+            expected_score
+        );
     }
 
     #[tokio::test]
@@ -7838,6 +11235,141 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_and_query_accepts_same_position_alternatives() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token in ["getusername", "get", "user", "name"] {
+            builder.tokens.add(token.to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+        }
+        // Doc 0 only has the split words. Doc 1 has both the complete
+        // identifier and split words. A grouped AND query should accept either
+        // `getusername` or `get` at position 0.
+        builder.posting_lists[1].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[3].add(0, PositionRecorder::Count(1));
+        builder.docs.append(100, 3);
+
+        builder.posting_lists[0].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[1].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[3].add(1, PositionRecorder::Count(1));
+        builder.docs.append(101, 4);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::code()).await;
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec![
+                "getusername".to_string(),
+                "get".to_string(),
+                "user".to_string(),
+                "name".to_string(),
+            ],
+            vec![0, 0, 1, 2],
+            DocType::Text,
+        ));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let (mut row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        row_ids.sort_unstable();
+        assert_eq!(row_ids, vec![100, 101]);
+    }
+
+    #[tokio::test]
+    async fn test_phrase_query_accepts_same_position_alternatives() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, true, TokenSetFormat::default());
+        for token in ["getusername", "get", "user", "name"] {
+            builder.tokens.add(token.to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(true));
+        }
+        // Doc 0 only has split words. Doc 1 has both the complete identifier
+        // and split words at the same position. Doc 2 has the terms but not as
+        // an exact phrase.
+        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(0, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[3].add(0, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(100, 3);
+
+        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(1, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[3].add(1, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(101, 3);
+
+        builder.posting_lists[0].add(2, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(2, PositionRecorder::Position(vec![2].into()));
+        builder.posting_lists[3].add(2, PositionRecorder::Position(vec![3].into()));
+        builder.docs.append(102, 3);
+
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(
+            &store,
+            vec![0],
+            InvertedIndexParams::code().with_position(true),
+        )
+        .await;
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec![
+                "getusername".to_string(),
+                "get".to_string(),
+                "user".to_string(),
+                "name".to_string(),
+            ],
+            vec![0, 0, 1, 2],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_phrase_slop(Some(0)),
+        );
+        let (mut row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        row_ids.sort_unstable();
+        assert_eq!(row_ids, vec![100, 101]);
+    }
+
     // Enough distinct tokens that `write_posting_lists` emits several posting-list
     // batches (the default batch size is 256 rows), exercising the restructured
     // producer and async send path.
@@ -8116,6 +11648,132 @@ mod tests {
         );
     }
 
+    /// Write one partition holding `variants` in order, with one
+    /// single-token doc per variant taken from `row_ids`.
+    async fn write_variant_partition(
+        store: &Arc<LanceIndexStore>,
+        partition_id: u64,
+        variants: &[&str],
+        row_ids: &[u64],
+    ) {
+        let mut builder = InnerBuilder::new(partition_id, false, TokenSetFormat::default());
+        for token in variants {
+            builder.tokens.add((*token).to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+        }
+        for (local_idx, row_id) in row_ids.iter().enumerate() {
+            builder.posting_lists[local_idx].add(local_idx as u32, PositionRecorder::Count(1));
+            builder.docs.append(*row_id, 1);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_expansion_cap_is_global_across_partitions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_variant_partition(&store, 0, &["alpha", "alphb"], &[100, 101]).await;
+        write_variant_partition(&store, 1, &["alphc", "alphd"], &[102, 103]).await;
+        write_test_metadata(&store, vec![0, 1], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let params = FtsSearchParams::new()
+            .with_fuzziness(Some(1))
+            .with_max_expansions(3);
+        let tokens = Tokens::new(vec!["alphx".to_owned()], DocType::Text);
+
+        let expanded = index.expand_fuzzy_tokens(&tokens, &params).unwrap();
+        let expanded_terms = (0..expanded.len())
+            .map(|idx| expanded.get_token(idx).to_owned())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            expanded_terms,
+            vec!["alpha".to_owned(), "alphb".to_owned(), "alphc".to_owned()],
+            "max_expansions must cap the whole query across partitions, \
+             in lexicographic order"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_results_independent_of_partition_shape() {
+        // The same four single-variant docs, laid out as one partition and
+        // as two. With a binding max_expansions the two shapes must still
+        // match the same documents with the same scores.
+        let single_dir = TempObjDir::default();
+        let single_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            single_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        write_variant_partition(
+            &single_store,
+            0,
+            &["alpha", "alphb", "alphc", "alphd"],
+            &[100, 101, 102, 103],
+        )
+        .await;
+        write_test_metadata(&single_store, vec![0], InvertedIndexParams::default()).await;
+
+        let split_dir = TempObjDir::default();
+        let split_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            split_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        write_variant_partition(&split_store, 0, &["alpha", "alphb"], &[100, 101]).await;
+        write_variant_partition(&split_store, 1, &["alphc", "alphd"], &[102, 103]).await;
+        write_test_metadata(&split_store, vec![0, 1], InvertedIndexParams::default()).await;
+
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_fuzziness(Some(1))
+                .with_max_expansions(3),
+        );
+
+        let mut results = Vec::new();
+        for store in [single_store, split_store] {
+            let cache = LanceCache::with_capacity(4096);
+            let index = InvertedIndex::load(store, None, &cache).await.unwrap();
+            let tokens = Arc::new(Tokens::new(vec!["alphx".to_owned()], DocType::Text));
+            let (row_ids, scores) = index
+                .bm25_search(
+                    tokens,
+                    params.clone(),
+                    Operator::Or,
+                    Arc::new(NoFilter),
+                    Arc::new(NoOpMetricsCollector),
+                    None,
+                )
+                .await
+                .unwrap();
+            let mut scored = row_ids.into_iter().zip(scores).collect::<Vec<_>>();
+            scored.sort_unstable_by_key(|(row_id, _)| *row_id);
+            results.push(scored);
+        }
+
+        assert_eq!(
+            results[0]
+                .iter()
+                .map(|(row_id, _)| *row_id)
+                .collect::<Vec<_>>(),
+            vec![100, 101, 102],
+            "a binding cap keeps the three lexicographically smallest variants"
+        );
+        assert_eq!(
+            results[0], results[1],
+            "fuzzy results must not depend on the partition shape"
+        );
+    }
+
     #[tokio::test]
     async fn test_fuzzy_and_scores_grouped_expansions_by_matched_token() {
         let tmpdir = TempObjDir::default();
@@ -8182,6 +11840,64 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[case::and(Operator::And)]
+    #[case::or(Operator::Or)]
+    #[tokio::test]
+    async fn test_grouped_scoring_keeps_exact_winner_outside_proxy_window(
+        #[case] operator: Operator,
+    ) {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("common".to_owned());
+        builder.tokens.add("rare".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        for doc_id in 0..3 {
+            builder.posting_lists[0].add(doc_id, PositionRecorder::Count(1));
+            builder.docs.append(100 + doc_id as u64, 1);
+        }
+        builder.posting_lists[1].add(3, PositionRecorder::Count(1));
+        builder.docs.append(103, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec!["common".to_owned(), "rare".to_owned()],
+            vec![0, 0],
+            DocType::Text,
+        ));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let (row_ids, _scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                operator,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(
+            row_ids,
+            vec![103],
+            "the rare term's exact IDF must win even when proxy scoring ranks it outside the old candidate cushion"
+        );
+    }
+
     #[tokio::test]
     async fn test_fuzzy_and_grouped_rescore_keeps_wand_limit_bounded() {
         let tmpdir = TempObjDir::default();
@@ -8459,7 +12175,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_update_preserves_loaded_v2_format_version() -> Result<()> {
+    async fn test_update_preserves_v2_format_version() -> Result<()> {
         let src_dir = TempObjDir::default();
         let dest_dir = TempObjDir::default();
         let src_store = Arc::new(LanceIndexStore::new(
@@ -8514,31 +12230,88 @@ mod tests {
         writer.finish_with_metadata(metadata).await.unwrap();
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(index.index_version(), format_version.index_version());
+        assert_eq!(index.format_version(), format_version);
+        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V2);
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("doc", DataType::Utf8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let docs = Arc::new(StringArray::from(vec![Some("hello again")]));
+        let row_ids = Arc::new(UInt64Array::from(vec![101u64]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?;
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
+        let created = index
+            .update(Box::pin(stream), dest_store.as_ref(), None)
+            .await?;
+
+        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V2);
+
+        let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(updated.format_version(), format_version);
+        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V2);
+        assert_eq!(updated.partitions.len(), 2);
+        for partition in &updated.partitions {
+            assert_eq!(
+                partition.inverted_list.posting_tail_codec(),
+                posting_tail_codec
+            );
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_block_size_256_writes_v3_metadata_and_index_version() -> Result<()> {
+        let src_dir = TempObjDir::default();
+        let dest_dir = TempObjDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let params = InvertedIndexParams::default().block_size(256)?;
+        let format_version = params.resolved_format_version();
+        assert_eq!(format_version, InvertedListFormatVersion::V3);
+
+        let mut partition = InnerBuilder::new_with_format_version_and_block_size(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+            params.posting_block_size(),
+        );
+        partition.tokens.add("hello".to_owned());
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            params.posting_block_size(),
+        );
+        posting_list.add(0, PositionRecorder::Count(1));
+        partition.posting_lists.push(posting_list);
+        partition.docs.append(100, 1);
+        partition.write(src_store.as_ref()).await?;
+
+        write_test_metadata(&src_store, vec![0], params).await;
+
+        let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(index.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V3);
 
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("doc", DataType::Utf8, true),
-            Field::new(ROW_ID, DataType::UInt64, false),
-        ]));
-        let docs = Arc::new(StringArray::from(vec![Some("hello again")]));
-        let row_ids = Arc::new(UInt64Array::from(vec![101u64]));
-        let batch = RecordBatch::try_new(schema.clone(), vec![docs, row_ids])?;
-        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
         let created = index
-            .update(Box::pin(stream), dest_store.as_ref(), None)
+            .update(empty_doc_stream(), dest_store.as_ref(), None)
             .await?;
-
-        assert_eq!(created.index_version, format_version.index_version());
+        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V3);
 
         let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(updated.index_version(), format_version.index_version());
-        assert_eq!(updated.partitions.len(), 2);
-        for partition in &updated.partitions {
-            assert_eq!(
-                partition.inverted_list.posting_tail_codec(),
-                posting_tail_codec
-            );
-        }
+        assert_eq!(updated.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V3);
 
         Ok(())
     }
@@ -8560,12 +12333,13 @@ mod tests {
 
         let index = write_single_partition_index(
             src_store,
-            InvertedIndexParams::default(),
+            InvertedIndexParams::default().format_version(InvertedListFormatVersion::V2),
             TokenSetFormat::Arrow,
             "hello",
             100,
         )
         .await?;
+        assert_eq!(index.index_version(), 0);
         let created = InvertedIndex::merge_segments(
             &[index],
             empty_doc_stream(),
@@ -8577,6 +12351,7 @@ mod tests {
 
         assert_eq!(created.index_version, 0);
         let merged = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(merged.index_version(), 0);
         assert_eq!(merged.token_set_format, TokenSetFormat::Arrow);
 
         let tokens = Arc::new(Tokens::new(vec!["hello".to_string()], DocType::Text));
@@ -8591,6 +12366,54 @@ mod tests {
         Ok(())
     }
 
+    #[rstest::rstest]
+    #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)]
+    #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)]
+    #[case::v3_128(InvertedListFormatVersion::V3, LEGACY_BLOCK_SIZE)]
+    #[case::v3_256(InvertedListFormatVersion::V3, 256)]
+    #[tokio::test]
+    async fn test_merge_segments_preserves_format_version(
+        #[case] format_version: InvertedListFormatVersion,
+        #[case] block_size: usize,
+    ) -> Result<()> {
+        let src_dir = TempObjDir::default();
+        let dest_dir = TempObjDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)?
+            .format_version(format_version);
+
+        let index =
+            write_single_partition_index(src_store, params, TokenSetFormat::Fst, "hello", 100)
+                .await?;
+        assert_eq!(index.format_version(), format_version);
+
+        let created = InvertedIndex::merge_segments(
+            &[index],
+            empty_doc_stream(),
+            dest_store.as_ref(),
+            None,
+            crate::progress::noop_progress(),
+        )
+        .await?;
+        assert_eq!(created.index_version, format_version.index_version());
+
+        let merged = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(merged.format_version(), format_version);
+        assert_eq!(merged.index_version(), format_version.index_version());
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_merge_segments_uses_memory_limit_for_old_partitions() -> Result<()> {
         let src_dir_1 = TempObjDir::default();
@@ -8764,6 +12587,60 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn flat_bm25_skips_zero_token_documents_from_corpus_stats() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(UInt64Array::from(vec![0_u64, 1, 2, 3, 4, 5])) as ArrayRef,
+                Arc::new(StringArray::from(vec![
+                    Some(""),
+                    Some("   "),
+                    Some("the"),
+                    Some("overlength"),
+                    None,
+                    Some("hello"),
+                ])) as ArrayRef,
+            ],
+        )
+        .unwrap();
+        let params = InvertedIndexParams::new("whitespace".to_string(), Language::English)
+            .remove_stop_words(true)
+            .stem(false)
+            .max_token_length(Some(6));
+        let query_tokens = Arc::new(Tokens::new(vec!["hello".to_string()], DocType::Text));
+
+        let counted_input = tokenize_and_count(
+            stream::iter(vec![Ok(batch)]),
+            params.build().unwrap(),
+            query_tokens.clone(),
+            1,
+            None,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(counted_input.num_rows(), 1);
+        assert_eq!(
+            counted_input[ROW_ID].as_primitive::<UInt64Type>().values(),
+            &[5]
+        );
+        let scorer = initialize_scorer(None, query_tokens.as_ref(), &counted_input);
+        let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)]));
+        assert_eq!(scorer.total_tokens, 1);
+        assert_eq!(scorer.num_docs(), 1);
+        assert_eq!(scorer.num_docs_containing_token("hello"), 1);
+        assert_eq!(scorer.avg_doc_length(), expected_scorer.avg_doc_length());
+        assert_eq!(
+            scorer.query_weight("hello"),
+            expected_scorer.query_weight("hello")
+        );
+    }
+
     #[tokio::test]
     async fn flat_bm25_search_uses_full_document_length_for_normalization() {
         let schema = Arc::new(Schema::new(vec![
@@ -8815,58 +12692,165 @@ mod tests {
         );
     }
 
-    /// An [`IndexReader`] wrapper that hides the posting-group-offsets schema
-    /// metadata key, so a [`PostingListReader`] opened on it takes the
-    /// pre-grouping per-token fallback path (issue #7040).
-    struct GroupKeyStrippingReader {
-        inner: Arc<dyn IndexReader>,
-        schema: lance_core::datatypes::Schema,
+    #[tokio::test]
+    async fn flat_bm25_search_treats_string_lists_as_row_documents() {
+        let mut docs_builder =
+            GenericListBuilder::<i32, _>::new(GenericStringBuilder::<i32>::new());
+        docs_builder.values().append_value("alpha");
+        docs_builder.values().append_value("alpha beta");
+        docs_builder.append(true);
+        docs_builder.values().append_value("beta");
+        docs_builder.append(true);
+        docs_builder.append(true);
+        docs_builder.values().append_null();
+        docs_builder.append(true);
+        docs_builder.append(false);
+
+        let docs = Arc::new(docs_builder.finish()) as ArrayRef;
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("text", docs.data_type().clone(), true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3, 4])) as ArrayRef,
+                docs,
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer: Box<dyn LanceTokenizer> = Box::new(TextTokenizer::new(
+            TextAnalyzer::builder(SimpleTokenizer::default()).build(),
+        ));
+
+        let result_stream = flat_bm25_search_stream_with_metrics(
+            input,
+            "text".to_string(),
+            "alpha".to_string(),
+            tokenizer,
+            None,
+            100,
+            None,
+        )
+        .await
+        .unwrap();
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let row_ids = scored[ROW_ID].as_primitive::<UInt64Type>();
+
+        assert_eq!(row_ids.values(), &[0]);
     }
 
-    impl GroupKeyStrippingReader {
-        fn new(inner: Arc<dyn IndexReader>) -> Self {
-            let mut schema = inner.schema().clone();
-            schema.metadata.remove(POSTING_GROUP_OFFSETS_BUF_KEY);
-            Self { inner, schema }
-        }
+    #[tokio::test]
+    async fn flat_bm25_search_code_and_uses_position_groups() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "get user name",
+                    "getUserName",
+                    "get user",
+                    "username",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer = InvertedIndexParams::code()
+            .split_identifiers(true)
+            .build()
+            .unwrap();
+
+        let result_stream = flat_bm25_search_stream_with_metrics_and_operator(
+            input,
+            "code".to_string(),
+            "getUserName".to_string(),
+            tokenizer,
+            None,
+            100,
+            Operator::And,
+            None,
+        )
+        .await
+        .unwrap();
+
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let mut row_ids = scored[ROW_ID]
+            .as_primitive::<UInt64Type>()
+            .values()
+            .to_vec();
+        row_ids.sort_unstable();
+
+        assert_eq!(row_ids, vec![0, 1]);
     }
 
-    #[async_trait]
-    impl IndexReader for GroupKeyStrippingReader {
-        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
-            self.inner.read_record_batch(n, batch_size).await
-        }
-        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
-            self.inner.read_global_buffer(index).await
-        }
-        async fn read_range(
-            &self,
-            range: std::ops::Range<usize>,
-            projection: Option<&[&str]>,
-        ) -> Result<RecordBatch> {
-            self.inner.read_range(range, projection).await
-        }
-        async fn num_batches(&self, batch_size: u64) -> u32 {
-            self.inner.num_batches(batch_size).await
-        }
-        fn num_rows(&self) -> usize {
-            self.inner.num_rows()
-        }
-        fn schema(&self) -> &lance_core::datatypes::Schema {
-            &self.schema
-        }
+    #[tokio::test]
+    async fn flat_bm25_search_code_and_counts_repeated_subwords() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1])),
+                Arc::new(StringArray::from(vec![
+                    "pub fn edge_flat_generic_return<T>() -> Result<T, EdgeFlatError> where T: TryFrom<String> { todo!() }",
+                    "pub fn edge_flat_generic_return<T>() -> Result<T> { todo!() }",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer = InvertedIndexParams::code().build().unwrap();
+
+        let result_stream = flat_bm25_search_stream_with_metrics_and_operator(
+            input,
+            "code".to_string(),
+            "edge_flat_generic_return TryFrom EdgeFlatError Result".to_string(),
+            tokenizer,
+            None,
+            100,
+            Operator::And,
+            None,
+        )
+        .await
+        .unwrap();
+
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let row_ids = scored[ROW_ID].as_primitive::<UInt64Type>().values();
+
+        assert_eq!(row_ids, &[0]);
     }
 
     fn posting_entries(posting: &PostingList) -> Vec<(u64, u32)> {
         posting.iter().map(|(doc, freq, _)| (doc, freq)).collect()
     }
 
-    /// The grouped read path and the legacy per-token fallback must return
-    /// identical posting lists for every token, including at group
-    /// boundaries. Builds a single v2 partition that spans several groups,
-    /// then reads it both with and without the group offsets present.
+    /// Runtime synthetic grouping must return correct posting lists for every
+    /// token, including across synthetic group boundaries.
     #[tokio::test]
-    async fn test_posting_list_fallback_matches_grouped() {
+    async fn test_posting_list_synthetic_grouping_reads_group_boundaries() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -8874,15 +12858,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // A small token cap forces several groups regardless of the default,
-        // so the comparison exercises the partition_point math at group
-        // boundaries.
-        let num_tokens = 150u32;
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_tokens {
             builder.tokens.add(format!("t{t}"));
             let mut pl = PostingListBuilder::new(false);
@@ -8894,36 +12871,27 @@ mod tests {
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
         let cache = LanceCache::no_cache();
-        let grouped = PostingListReader::try_new(reader.clone(), &cache)
-            .await
-            .unwrap();
-        assert!(
-            grouped.group_starts.as_ref().is_some_and(|s| s.len() > 1),
-            "fixture should span multiple groups",
-        );
-
-        let stripped: Arc<dyn IndexReader> = Arc::new(GroupKeyStrippingReader::new(reader));
-        let fallback = PostingListReader::try_new(stripped, &cache).await.unwrap();
+        let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
         assert!(
-            fallback.group_starts.is_none(),
-            "stripped reader must take the per-token fallback path",
+            matches!(
+                &posting_reader.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "v2 reader must synthesize runtime posting groups",
         );
 
         let metrics = NoOpMetricsCollector;
         for token in 0..num_tokens {
-            let g = grouped.posting_list(token, false, &metrics).await.unwrap();
-            let f = fallback.posting_list(token, false, &metrics).await.unwrap();
-            assert_eq!(
-                posting_entries(&g),
-                posting_entries(&f),
-                "grouped vs fallback mismatch for token {token}",
-            );
-            assert_eq!(g.len(), f.len(), "length mismatch for token {token}");
+            let posting = posting_reader
+                .posting_list(token, false, &metrics)
+                .await
+                .unwrap();
             assert_eq!(
-                g.max_score(),
-                f.max_score(),
-                "max_score mismatch for token {token}",
+                posting_entries(&posting),
+                vec![(token as u64, 1)],
+                "synthetic grouping mismatch for token {token}",
             );
+            assert_eq!(posting.len(), 1, "length mismatch for token {token}");
         }
     }
 
@@ -8941,14 +12909,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // Small token cap so the partition spans several groups regardless of
-        // the default, exercising every group boundary including the last.
-        let num_tokens = 150u32;
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_tokens {
             builder.tokens.add(format!("t{t}"));
             let mut pl = PostingListBuilder::new(false);
@@ -8964,11 +12926,11 @@ mod tests {
         let cache = LanceCache::with_capacity(1 << 20);
         let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
         assert!(
-            posting_reader
-                .group_starts
-                .as_ref()
-                .is_some_and(|s| s.len() > 1),
-            "fixture should span multiple groups",
+            matches!(
+                &posting_reader.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "v2 reader should use runtime synthetic groups",
         );
 
         posting_reader
@@ -8981,7 +12943,11 @@ mod tests {
             assert!(
                 posting_reader
                     .index_cache
-                    .get_with_key(&PostingListGroupKey { start, end })
+                    .get_with_key(&posting_list_group_cache_key(
+                        start,
+                        end,
+                        posting_reader.has_impacts,
+                    ))
                     .await
                     .is_some(),
                 "prewarm did not populate group [{start}, {end}) that the read \
@@ -8998,10 +12964,10 @@ mod tests {
         );
     }
 
-    /// An empty partition writes no group-offsets buffer, so its reader takes
-    /// the per-token fallback path (issue #7040).
+    /// An empty partition has no synthetic groups because there are no token
+    /// rows to cache.
     #[tokio::test]
-    async fn test_empty_partition_has_no_group_offsets() {
+    async fn test_empty_partition_has_no_synthetic_groups() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -9013,28 +12979,20 @@ mod tests {
         builder.write(store.as_ref()).await.unwrap();
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
-        assert!(
-            !reader
-                .schema()
-                .metadata
-                .contains_key(POSTING_GROUP_OFFSETS_BUF_KEY),
-            "empty partition must not write the group-offsets metadata key",
-        );
-
         let posting_reader = PostingListReader::try_new(reader, &LanceCache::no_cache())
             .await
             .unwrap();
         assert!(
-            posting_reader.group_starts.is_none(),
-            "reader for an empty partition must use the per-token fallback path",
+            matches!(&posting_reader.grouping, PostingGrouping::None),
+            "reader for an empty partition must not create cache groups",
         );
         assert!(posting_reader.is_empty());
     }
 
-    /// A posting list that alone exceeds the group target lands in its own
-    /// `[t, t+1)` group (the clamp case) and reads back intact (issue #7040).
+    /// A large posting list can share a runtime synthetic group with neighbors;
+    /// grouping is token-count based and should still read every member intact.
     #[tokio::test]
-    async fn test_oversized_term_is_own_group_on_read() {
+    async fn test_large_posting_reads_inside_synthetic_group() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -9042,14 +13000,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // A tiny byte target so a modest posting trips the clamp without
-        // needing a huge fixture; the surrounding tiny terms regroup after it.
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 50,
-            max_tokens: 1000,
-        };
-        let big_docs = 30u32;
+        let big_docs = (BLOCK_SIZE * 3 + 5) as u32;
         builder.tokens.add("big".to_owned());
         let mut big = PostingListBuilder::new(false);
         for d in 0..big_docs {
@@ -9071,11 +13023,12 @@ mod tests {
         let posting_reader = PostingListReader::try_new(reader, &LanceCache::no_cache())
             .await
             .unwrap();
+        let expected_end = runtime_posting_group_tokens().min(5) as u32;
 
         assert_eq!(
             posting_reader.group_range_for_token(0),
-            Some((0, 1)),
-            "an oversized term must occupy its own single-row group",
+            Some((0, expected_end)),
+            "runtime synthetic grouping should group by token count, not posting bytes",
         );
         let big = posting_reader
             .posting_list(0, false, &NoOpMetricsCollector)
@@ -9090,11 +13043,11 @@ mod tests {
         assert_eq!(tiny.len(), 1);
     }
 
-    /// When the group offsets are absent, prewarm populates per-token
-    /// `PostingListKey` entries (the fallback path), matching what the read
-    /// path then looks up (issue #7040).
+    /// Non-empty v2 indexes should prewarm synthetic `PostingListGroupKey`
+    /// entries, matching what the read path then looks up without persisted
+    /// grouping metadata.
     #[tokio::test]
-    async fn test_prewarm_fallback_populates_per_token_entries() {
+    async fn test_prewarm_synthetic_grouping_populates_group_entries() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -9114,10 +13067,12 @@ mod tests {
         builder.write(store.as_ref()).await.unwrap();
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
-        let stripped: Arc<dyn IndexReader> = Arc::new(GroupKeyStrippingReader::new(reader));
         let cache = LanceCache::with_capacity(1 << 20);
-        let posting_reader = PostingListReader::try_new(stripped, &cache).await.unwrap();
-        assert!(posting_reader.group_starts.is_none());
+        let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
+        assert!(matches!(
+            &posting_reader.grouping,
+            PostingGrouping::SyntheticFixed { .. }
+        ));
 
         posting_reader
             .prewarm_posting_lists(false, 2)
@@ -9125,13 +13080,34 @@ mod tests {
             .unwrap();
 
         for token_id in 0..num_tokens {
+            let (start, end) = posting_reader.group_range_for_token(token_id).unwrap();
+            let group = posting_reader
+                .index_cache
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    posting_reader.has_impacts,
+                ))
+                .await
+                .unwrap_or_else(|| {
+                    panic!(
+                        "synthetic prewarm should populate group [{start}, {end}) for token {token_id}"
+                    )
+                });
+            assert!(
+                group.is_packed(),
+                "no-position synthetic prewarm should insert a packed group"
+            );
             assert!(
                 posting_reader
                     .index_cache
-                    .get_with_key(&PostingListKey { token_id })
+                    .get_with_key(&posting_list_cache_key(
+                        token_id,
+                        posting_reader.has_impacts,
+                    ))
                     .await
-                    .is_some(),
-                "fallback prewarm should populate per-token entry {token_id}",
+                    .is_none(),
+                "synthetic prewarm should not populate per-token entry {token_id}",
             );
         }
     }
@@ -9148,15 +13124,11 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // 130 rare tokens (one doc each) plus one common token in every doc; a
-        // small token cap spreads them across several groups so scoring must
-        // index into the right group slot.
-        let num_rare = 130u32;
+        // Rare tokens (one doc each) plus one common token in every doc. The
+        // token count exceeds the runtime group size so scoring must index
+        // into the right synthetic group slot.
+        let num_rare = runtime_posting_group_tokens() as u32 + 2;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_rare {
             builder.tokens.add(format!("t{t}"));
             builder.posting_lists.push(PostingListBuilder::new(false));
@@ -9203,7 +13175,7 @@ mod tests {
                 index
                     .bm25_search(
                         Arc::new(Tokens::new(vec![term], DocType::Text)),
-                        Arc::new(FtsSearchParams::new().with_limit(Some(200))),
+                        Arc::new(FtsSearchParams::new().with_limit(Some(num_rare as usize))),
                         Operator::Or,
                         Arc::new(NoFilter),
                         Arc::new(NoOpMetricsCollector),
@@ -9214,8 +13186,13 @@ mod tests {
             }
         };
 
-        let (rows_70, _) = query("t70").await;
-        assert_eq!(rows_70, vec![1070], "rare token must map to its single doc");
+        let rare_query_id = num_rare / 2;
+        let (rare_rows, _) = query(&format!("t{rare_query_id}")).await;
+        assert_eq!(
+            rare_rows,
+            vec![1000 + rare_query_id as u64],
+            "rare token must map to its single doc",
+        );
 
         // Cold vs warm cache must agree for the common (large) token.
         let (cold_rows, cold_scores) = query("common").await;
diff --git a/rust/lance-index/src/scalar/inverted/iter.rs b/rust/lance-index/src/scalar/inverted/iter.rs
index dc07b15769c..3755fa41d80 100644
--- a/rust/lance-index/src/scalar/inverted/iter.rs
+++ b/rust/lance-index/src/scalar/inverted/iter.rs
@@ -6,10 +6,9 @@ use arrow_array::{Array, LargeBinaryArray};
 
 use super::{
     CompressedPositionStorage, PostingList, PostingTailCodec,
-    builder::BLOCK_SIZE,
     encoding::{
-        decode_position_stream_block, decompress_positions, decompress_posting_block,
-        decompress_posting_remainder,
+        MAX_POSTING_BLOCK_SIZE, decode_position_stream_block, decompress_positions,
+        decompress_posting_block, decompress_posting_remainder,
     },
 };
 
@@ -28,6 +27,7 @@ impl<'a> PostingListIterator<'a> {
                     posting.blocks.clone(),
                     posting.posting_tail_codec,
                     posting.positions.clone(),
+                    posting.block_size,
                 )))
             }
         }
@@ -82,13 +82,14 @@ pub struct CompressedPostingListIterator {
     next_block_idx: usize,
     posting_tail_codec: PostingTailCodec,
     positions: Option<CompressedPositionStorage>,
+    block_size: usize,
     idx: usize,
     doc_ids: Vec<u32>,
     frequencies: Vec<u32>,
     doc_idx_in_block: usize,
     decoded_positions: Vec<u32>,
     position_offsets: Vec<usize>,
-    buffer: [u32; BLOCK_SIZE],
+    buffer: [u32; MAX_POSTING_BLOCK_SIZE],
 }
 
 impl CompressedPostingListIterator {
@@ -97,10 +98,11 @@ impl CompressedPostingListIterator {
         blocks: LargeBinaryArray,
         posting_tail_codec: PostingTailCodec,
         positions: Option<CompressedPositionStorage>,
+        block_size: usize,
     ) -> Self {
         debug_assert!(length > 0, "length: {}", length);
         debug_assert_eq!(
-            length.div_ceil(BLOCK_SIZE),
+            length.div_ceil(block_size),
             blocks.len(),
             "length: {}, num_blocks: {}",
             length,
@@ -108,18 +110,19 @@ impl CompressedPostingListIterator {
         );
 
         Self {
-            remainder: length % BLOCK_SIZE,
+            remainder: length % block_size,
             blocks,
             next_block_idx: 0,
             posting_tail_codec,
             positions,
+            block_size,
             idx: 0,
             doc_ids: Vec::new(),
             frequencies: Vec::new(),
             doc_idx_in_block: 0,
             decoded_positions: Vec::new(),
             position_offsets: Vec::new(),
-            buffer: [0; BLOCK_SIZE],
+            buffer: [0; MAX_POSTING_BLOCK_SIZE],
         }
     }
 }
@@ -163,6 +166,7 @@ impl Iterator for CompressedPostingListIterator {
                 compressed,
                 self.remainder,
                 self.posting_tail_codec,
+                self.block_size,
                 &mut self.doc_ids,
                 &mut self.frequencies,
             );
@@ -172,6 +176,7 @@ impl Iterator for CompressedPostingListIterator {
                 &mut self.buffer,
                 &mut self.doc_ids,
                 &mut self.frequencies,
+                self.block_size,
             );
         }
         self.doc_idx_in_block = 0;
diff --git a/rust/lance-index/src/scalar/inverted/lazy_docset.rs b/rust/lance-index/src/scalar/inverted/lazy_docset.rs
index 12b5c1ebd32..faf87dd6bac 100644
--- a/rust/lance-index/src/scalar/inverted/lazy_docset.rs
+++ b/rust/lance-index/src/scalar/inverted/lazy_docset.rs
@@ -14,18 +14,29 @@
 //! demand and cache. Wand scoring still needs per-doc num_tokens, but
 //! only partitions that actually contribute hits pay
 //! `ensure_num_tokens_loaded`/`ensure_loaded`.
+//!
+//! For modern partitions the `doc_id -> row_id` column has exactly one
+//! home: the [`DocRowIdsKey`] index-cache entry. No `DocSet` keeps a
+//! long-lived copy or reference, so evicting the entry really frees the
+//! memory; borrowers reload it single-flight. Legacy and frag-reuse
+//! partitions rewrite the mapping at load and keep a private owned copy.
 
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use arrow::array::AsArray;
 use arrow::datatypes::{UInt32Type, UInt64Type};
-use arrow_array::{UInt32Array, UInt64Array};
+use arrow_array::{Array, UInt32Array, UInt64Array};
 use lance_core::ROW_ID;
 use lance_core::Result;
+use lance_core::cache::{CacheKey, WeakLanceCache};
+use lance_core::deepsize::DeepSizeOf;
 use tokio::sync::OnceCell;
 
 use crate::scalar::RowIdRemapper;
 use crate::scalar::inverted::index::{DocSet, NUM_TOKEN_COL};
+use crate::scalar::inverted::query::Operator;
+use crate::scalar::inverted::wand::should_flat_search;
 use crate::scalar::{IndexReader, IndexStore};
 use lance_select::mask::RowAddrMask;
 
@@ -49,36 +60,81 @@ pub struct LoadedDocSet {
     total_tokens: u64,
 }
 
+/// Atomically published num-tokens state for deferred scoring.
+///
+/// Keeping the Arrow column and the zero-copy `DocSet` view that carries its
+/// total in one `OnceCell` prevents cancellation from exposing a partially
+/// initialized scoring cache.
+struct NumTokensSnapshot {
+    column: Arc<UInt32Array>,
+    docs: Arc<DocSet>,
+}
+
+/// A partition's full `doc_id -> row_id` column, stored as its own
+/// index-cache entry so the cache weighs it at insert time and can
+/// evict it independently of the partition object. Keeping it out of
+/// the partition's lazily-populated fields keeps the memory visible
+/// to capacity accounting.
+#[derive(Debug)]
+pub struct CachedDocRowIds {
+    pub row_ids: Arc<UInt64Array>,
+}
+
+impl DeepSizeOf for CachedDocRowIds {
+    fn deep_size_of_children(&self, _ctx: &mut lance_core::deepsize::Context) -> usize {
+        self.row_ids.len() * std::mem::size_of::<u64>()
+    }
+}
+
+/// Cache key for [`CachedDocRowIds`], scoped per partition.
+#[derive(Debug, Clone)]
+pub struct DocRowIdsKey {
+    pub partition_id: u64,
+}
+
+impl CacheKey for DocRowIdsKey {
+    type ValueType = CachedDocRowIds;
+
+    fn key(&self) -> Cow<'_, str> {
+        format!("doc-row-ids-{}", self.partition_id).into()
+    }
+
+    fn type_name() -> &'static str {
+        "DocRowIds"
+    }
+}
+
 /// Store-backed DocSet view that loads on demand and caches.
 ///
 /// Holds the [`IndexStore`] and docs-file path rather than an open
 /// [`IndexReader`], so a cached partition does not pin a docs-file
 /// handle for its whole lifetime. The reader is re-opened on demand
 /// inside each column accessor and dropped when that read completes;
-/// because the resulting buffers are cached in the `OnceCell`s below,
-/// a contributing partition re-opens only on a cold miss, and a
-/// partition that never scores never opens the docs file at all after
-/// construction.
+/// because the resulting buffers are cached (num_tokens and the resident
+/// scoring set in the `OnceCell`s below, the row_id column as its own
+/// index-cache entry), a contributing partition re-opens only on a
+/// cold miss, and a partition that never scores never opens the docs
+/// file at all after construction.
 pub struct DeferredDocSet {
     store: Arc<dyn IndexStore>,
     docs_path: String,
+    /// Scopes this partition's [`DocRowIdsKey`] in the index cache.
+    partition_id: u64,
+    /// The index cache holding the row_id column entry.
+    index_cache: WeakLanceCache,
     is_legacy: bool,
     frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+    /// 256-document-block partitions score with quantized document lengths; the
+    /// flag is applied to every `DocSet` this deferred set materializes.
+    quantized_scoring: bool,
     /// Doc count cached at construction so `len()` stays sync + IO-free.
     num_rows: usize,
-    /// `sum(num_tokens)` cached on first compute.
-    total_tokens: OnceCell<u64>,
-    /// `NUM_TOKEN_COL` arrow buffer cached on first read.
-    num_tokens_col: OnceCell<Arc<UInt32Array>>,
-    /// `ROW_ID` arrow buffer cached on first read.
-    row_ids_col: OnceCell<Arc<UInt64Array>>,
-    /// Full DocSet, materialized on first `ensure_loaded`.
-    full: OnceCell<Arc<DocSet>>,
-    /// num_tokens-only DocSet, materialized on first
-    /// `ensure_num_tokens_loaded`. Cached because wand scoring calls this
-    /// once per query per partition; rebuilding it copied the whole
-    /// num_tokens column (tens of MB per partition) on every query.
-    tokens_only: OnceCell<Arc<DocSet>>,
+    /// `NUM_TOKEN_COL` and its zero-copy scoring view carrying the cached sum,
+    /// published together on first read.
+    num_tokens: OnceCell<NumTokensSnapshot>,
+    /// Modern partitions: num_tokens + `inv`, no row_ids (the column stays
+    /// in the cache entry). Legacy / frag reuse: the full rewritten DocSet.
+    resident: OnceCell<Arc<DocSet>>,
 }
 
 impl std::fmt::Debug for LazyDocSet {
@@ -92,58 +148,64 @@ impl std::fmt::Debug for LazyDocSet {
             Self::Deferred(d) => f
                 .debug_struct("LazyDocSet::Deferred")
                 .field("num_rows", &d.num_rows)
-                .field("total_tokens_loaded", &d.total_tokens.initialized())
-                .field("full_loaded", &d.full.initialized())
+                .field(
+                    "total_tokens_loaded",
+                    &(d.num_tokens.initialized() || d.resident.initialized()),
+                )
+                .field("num_tokens_loaded", &d.num_tokens.initialized())
+                .field("resident_loaded", &d.resident.initialized())
                 .finish(),
         }
     }
 }
 
-impl lance_core::deepsize::DeepSizeOf for LazyDocSet {
+impl DeepSizeOf for LazyDocSet {
     fn deep_size_of_children(&self, ctx: &mut lance_core::deepsize::Context) -> usize {
         match self {
             Self::Loaded(l) => l.docs.deep_size_of_children(ctx),
             Self::Deferred(d) => {
-                d.full
+                d.resident
                     .get()
                     .map(|d| d.deep_size_of_children(ctx))
                     .unwrap_or(0)
-                    + d.tokens_only
-                        .get()
-                        .map(|d| d.deep_size_of_children(ctx))
-                        .unwrap_or(0)
-                    + d.num_tokens_col
+                    + d.num_tokens
                         .get()
-                        .map(|arr| arr.len() * std::mem::size_of::<u32>())
-                        .unwrap_or(0)
-                    + d.row_ids_col
-                        .get()
-                        .map(|arr| arr.len() * std::mem::size_of::<u64>())
+                        .map(|snapshot| {
+                            let arr: &dyn Array = snapshot.column.as_ref();
+                            snapshot.docs.deep_size_of_children(ctx)
+                                + arr.deep_size_of_children(ctx)
+                        })
                         .unwrap_or(0)
+                // The row_id column is not a field here: it lives in the
+                // index cache as its own weighed entry (`DocRowIdsKey`).
             }
         }
     }
 }
 
 impl LazyDocSet {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         store: Arc<dyn IndexStore>,
         docs_path: String,
+        partition_id: u64,
+        index_cache: WeakLanceCache,
         num_rows: usize,
         is_legacy: bool,
         frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        quantized_scoring: bool,
     ) -> Self {
         Self::Deferred(Box::new(DeferredDocSet {
             store,
             docs_path,
+            partition_id,
+            index_cache,
             is_legacy,
             frag_reuse_index,
+            quantized_scoring,
             num_rows,
-            total_tokens: OnceCell::new(),
-            num_tokens_col: OnceCell::new(),
-            row_ids_col: OnceCell::new(),
-            full: OnceCell::new(),
-            tokens_only: OnceCell::new(),
+            num_tokens: OnceCell::new(),
+            resident: OnceCell::new(),
         }))
     }
 
@@ -174,7 +236,15 @@ impl LazyDocSet {
     pub fn total_tokens_cached(&self) -> Option<u64> {
         match self {
             Self::Loaded(l) => Some(l.total_tokens),
-            Self::Deferred(d) => d.total_tokens.get().copied(),
+            Self::Deferred(d) => d
+                .resident
+                .get()
+                .map(|docs| docs.total_tokens_num())
+                .or_else(|| {
+                    d.num_tokens
+                        .get()
+                        .map(|snapshot| snapshot.docs.total_tokens_num())
+                }),
         }
     }
 
@@ -197,7 +267,10 @@ impl LazyDocSet {
         }
     }
 
-    /// Materialize the full DocSet, including row_ids.
+    /// Make the scoring state resident: num_tokens, `inv`, and the row-ids
+    /// column loaded into its [`DocRowIdsKey`] cache entry. The returned
+    /// DocSet carries no row_ids for modern partitions, so nothing pins the
+    /// entry. Prewarm calls this.
     pub async fn ensure_loaded(&self) -> Result<Arc<DocSet>> {
         match self {
             Self::Loaded(l) => Ok(l.docs.clone()),
@@ -205,12 +278,30 @@ impl LazyDocSet {
         }
     }
 
+    /// A fully-owned DocSet, row_ids included, for rebuild paths that
+    /// mutate it. Never stashed, so it does not pin the cache entry.
+    pub async fn owned_docset(&self) -> Result<DocSet> {
+        match self {
+            Self::Loaded(l) => Ok((*l.docs).clone()),
+            Self::Deferred(d) => {
+                if d.is_legacy || d.frag_reuse_index.is_some() {
+                    return Ok((*d.ensure_loaded().await?).clone());
+                }
+                let (snapshot, row_ids) =
+                    futures::try_join!(d.num_tokens_snapshot(), d.row_ids_column())?;
+                let mut docs =
+                    DocSet::from_columns(row_ids.as_ref(), snapshot.column.as_ref(), false, None)?;
+                docs.set_quantized_scoring(d.quantized_scoring);
+                Ok(docs)
+            }
+        }
+    }
+
     /// Materialize a DocSet that carries num_tokens but no row_ids.
     /// Used by the deferred-row_id scoring path; the per-partition
     /// caller resolves surviving doc_ids -> row_ids post-wand via
-    /// [`Self::resolve_row_ids`]. The result is NOT cached on the
-    /// LazyDocSet -- a later `ensure_loaded` must still produce a
-    /// full DocSet.
+    /// [`Self::resolve_row_ids`]. The tokens-only result is cached separately;
+    /// a later `ensure_loaded` must still produce a full DocSet.
     pub async fn ensure_num_tokens_loaded(&self) -> Result<Arc<DocSet>> {
         match self {
             Self::Loaded(l) => Ok(l.docs.clone()),
@@ -218,16 +309,36 @@ impl LazyDocSet {
         }
     }
 
-    /// Pick the right DocSet shape for a wand walk under `mask`:
-    /// the num_tokens-only deferred form when the mask is trivial
-    /// AND no FragReuseIndex needs to filter row_ids; otherwise the
-    /// full DocSet. Encapsulates the policy so callers don't have to
-    /// rederive the conditions for the targeted-read fast path.
-    pub async fn docs_for_wand(&self, mask: &RowAddrMask) -> Result<Arc<DocSet>> {
-        if mask.is_select_all() && !self.has_frag_reuse_remap() {
-            self.ensure_num_tokens_loaded().await
-        } else {
-            self.ensure_loaded().await
+    /// Pick the DocSet shape for a wand walk: trivial mask → num_tokens
+    /// only; legacy / frag reuse → owned rewritten DocSet; flat-shaped mask
+    /// (same [`should_flat_search`] predicate wand uses) → resident set with
+    /// `inv`; any other mask → per-query view borrowing the cache entry.
+    pub async fn docs_for_wand(
+        &self,
+        operator: Operator,
+        mask: &RowAddrMask,
+    ) -> Result<Arc<DocSet>> {
+        match self {
+            Self::Loaded(l) => Ok(l.docs.clone()),
+            Self::Deferred(d) => {
+                if mask.is_select_all() && !self.has_frag_reuse_remap() {
+                    return self.ensure_num_tokens_loaded().await;
+                }
+                if d.is_legacy
+                    || d.frag_reuse_index.is_some()
+                    || should_flat_search(operator, mask, d.num_rows as u64)
+                {
+                    return self.ensure_loaded().await;
+                }
+                let (snapshot_docs, row_ids) = {
+                    let (snapshot, row_ids) =
+                        futures::try_join!(d.num_tokens_snapshot(), d.row_ids_column())?;
+                    (snapshot.docs.clone(), row_ids)
+                };
+                Ok(Arc::new(
+                    snapshot_docs.with_shared_row_ids(row_ids.values().clone()),
+                ))
+            }
         }
     }
 
@@ -254,109 +365,146 @@ impl DeferredDocSet {
     }
 
     async fn total_tokens_num(&self) -> Result<u64> {
-        if let Some(v) = self.total_tokens.get() {
-            return Ok(*v);
+        if let Some(resident) = self.resident.get() {
+            return Ok(resident.total_tokens_num());
         }
-        if let Some(full) = self.full.get() {
-            let v = full.total_tokens_num();
-            let _ = self.total_tokens.set(v);
-            return Ok(v);
-        }
-        let col = self.num_tokens_column().await?;
-        let sum: u64 = col.values().iter().map(|&n| n as u64).sum();
-        let _ = self.total_tokens.set(sum);
-        Ok(sum)
+        Ok(self.num_tokens_snapshot().await?.docs.total_tokens_num())
     }
 
-    async fn num_tokens_column(&self) -> Result<Arc<UInt32Array>> {
-        self.num_tokens_col
+    async fn num_tokens_snapshot(&self) -> Result<&NumTokensSnapshot> {
+        self.num_tokens
             .get_or_try_init(|| async {
                 let reader = self.reader().await?;
                 let batch = reader
                     .read_range(0..self.num_rows, Some(&[NUM_TOKEN_COL]))
                     .await?;
-                Result::Ok(Arc::new(
-                    batch[NUM_TOKEN_COL].as_primitive::<UInt32Type>().clone(),
-                ))
+                let column = Arc::new(batch[NUM_TOKEN_COL].as_primitive::<UInt32Type>().clone());
+                let total_tokens = column.values().iter().map(|&n| n as u64).sum();
+                let mut docs = DocSet::from_cached_num_tokens(column.as_ref(), total_tokens);
+                docs.set_quantized_scoring(self.quantized_scoring);
+                Result::Ok(NumTokensSnapshot {
+                    column,
+                    docs: Arc::new(docs),
+                })
             })
             .await
-            .cloned()
     }
 
+    /// The full `ROW_ID` column, loaded through the index cache as its own
+    /// weighed entry ([`DocRowIdsKey`]) rather than a field on this struct,
+    /// so the memory is accounted for at insert time and evictable under
+    /// pressure. Concurrent loads are deduped by the cache backend.
     async fn row_ids_column(&self) -> Result<Arc<UInt64Array>> {
-        self.row_ids_col
-            .get_or_try_init(|| async {
-                let reader = self.reader().await?;
-                let batch = reader.read_range(0..self.num_rows, Some(&[ROW_ID])).await?;
-                Result::Ok(Arc::new(batch[ROW_ID].as_primitive::<UInt64Type>().clone()))
-            })
-            .await
-            .cloned()
+        let store = self.store.clone();
+        let docs_path = self.docs_path.clone();
+        let num_rows = self.num_rows;
+        let cached = self
+            .index_cache
+            .get_or_insert_with_key(
+                DocRowIdsKey {
+                    partition_id: self.partition_id,
+                },
+                || async move {
+                    let reader = store.open_index_file(&docs_path).await?;
+                    let batch = reader.read_range(0..num_rows, Some(&[ROW_ID])).await?;
+                    Result::Ok(CachedDocRowIds {
+                        row_ids: Arc::new(batch[ROW_ID].as_primitive::<UInt64Type>().clone()),
+                    })
+                },
+            )
+            .await?;
+        Ok(cached.row_ids.clone())
     }
 
     async fn ensure_loaded(&self) -> Result<Arc<DocSet>> {
         let docs = self
-            .full
+            .resident
             .get_or_try_init(|| async {
-                // If the stats path already pulled NUM_TOKEN_COL,
-                // read only ROW_ID and rebuild from the two columns.
-                let docs = if self.num_tokens_col.get().is_some() {
-                    let num_tokens = self.num_tokens_column().await?;
-                    let row_ids = self.row_ids_column().await?;
-                    DocSet::from_columns(
-                        row_ids.as_ref(),
-                        num_tokens.as_ref(),
-                        self.is_legacy,
-                        self.frag_reuse_index.clone(),
-                    )?
-                } else {
+                let mut docs = if self.is_legacy || self.frag_reuse_index.is_some() {
+                    // Both rewrite the mapping, so keep a private owned copy.
                     DocSet::load(
                         self.reader().await?,
                         self.is_legacy,
                         self.frag_reuse_index.clone(),
                     )
                     .await?
+                } else {
+                    let (snapshot, row_ids) =
+                        futures::try_join!(self.num_tokens_snapshot(), self.row_ids_column())?;
+                    DocSet::from_cached_num_tokens_with_inv(
+                        row_ids.as_ref(),
+                        snapshot.column.as_ref(),
+                        snapshot.docs.total_tokens_num(),
+                    )
                 };
+                docs.set_quantized_scoring(self.quantized_scoring);
                 Result::Ok(Arc::new(docs))
             })
             .await?
             .clone();
-        let _ = self.total_tokens.set(docs.total_tokens_num());
         Ok(docs)
     }
 
     async fn ensure_num_tokens_loaded(&self) -> Result<Arc<DocSet>> {
-        if let Some(full) = self.full.get() {
-            return Ok(full.clone());
+        if let Some(resident) = self.resident.get() {
+            return Ok(resident.clone());
         }
-        let docs = self
-            .tokens_only
-            .get_or_try_init(|| async {
-                let num_tokens = self.num_tokens_column().await?;
-                Result::Ok(Arc::new(DocSet::from_num_tokens_only(num_tokens.as_ref())))
-            })
-            .await?
-            .clone();
-        let _ = self.total_tokens.set(docs.total_tokens_num());
-        Ok(docs)
+        Ok(self.num_tokens_snapshot().await?.docs.clone())
     }
 
     async fn resolve_row_ids(&self, doc_ids: &[u32]) -> Result<Vec<u64>> {
-        if let Some(full) = self.full.get()
-            && full.has_row_ids()
+        // Only legacy / frag-reuse residents carry (rewritten) row_ids;
+        // modern partitions read the cache entry, keeping it LRU-hot.
+        if let Some(resident) = self.resident.get()
+            && resident.has_row_ids()
         {
-            return Ok(doc_ids.iter().map(|&d| full.row_id(d)).collect());
-        }
-        if let Some(arr) = self.row_ids_col.get() {
-            return Ok(doc_ids.iter().map(|&d| arr.value(d as usize)).collect());
+            return Ok(doc_ids.iter().map(|&d| resident.row_id(d)).collect());
         }
-        let ranges: Vec<std::ops::Range<usize>> = doc_ids
-            .iter()
-            .map(|&d| d as usize..d as usize + 1)
-            .collect();
-        let reader = self.reader().await?;
-        let batch = reader.read_ranges(&ranges, Some(&[ROW_ID])).await?;
-        let arr = batch[ROW_ID].as_primitive::<UInt64Type>();
-        Ok((0..arr.len()).map(|i| arr.value(i)).collect())
+        let arr = self.row_ids_column().await?;
+        Ok(doc_ids.iter().map(|&d| arr.value(d as usize)).collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::scalar::lance_format::LanceIndexStore;
+    use lance_core::cache::LanceCache;
+    use lance_core::utils::tempfile::TempObjDir;
+    use lance_io::object_store::ObjectStore;
+
+    #[tokio::test]
+    async fn test_resident_docset_serves_num_tokens_reads() {
+        let temp_dir = TempObjDir::default();
+        let cache = Arc::new(LanceCache::no_cache());
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            temp_dir.clone(),
+            cache.clone(),
+        ));
+        let docs = LazyDocSet::new(
+            store,
+            "unused".to_owned(),
+            0,
+            WeakLanceCache::from(&cache),
+            3,
+            false,
+            None,
+            false,
+        );
+        assert_eq!(docs.total_tokens_cached(), None);
+
+        let row_ids = UInt64Array::from(vec![10, 20, 30]);
+        let num_tokens = UInt32Array::from(vec![3, 5, 8]);
+        let full = Arc::new(DocSet::from_columns(&row_ids, &num_tokens, false, None).unwrap());
+        let LazyDocSet::Deferred(deferred) = &docs else {
+            panic!("expected a deferred DocSet");
+        };
+        deferred.resident.set(full.clone()).unwrap();
+
+        let wand_docs = docs.ensure_num_tokens_loaded().await.unwrap();
+        assert!(Arc::ptr_eq(&wand_docs, &full));
+        assert_eq!(wand_docs.total_tokens_num(), 16);
+        assert_eq!(docs.total_tokens_cached(), Some(16));
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/scorer.rs b/rust/lance-index/src/scalar/inverted/scorer.rs
index eb7d78ff397..3a33a67ff7a 100644
--- a/rust/lance-index/src/scalar/inverted/scorer.rs
+++ b/rust/lance-index/src/scalar/inverted/scorer.rs
@@ -3,6 +3,7 @@
 
 use super::InvertedPartition;
 use std::collections::HashMap;
+use std::sync::Arc;
 
 // the Scorer trait is used to calculate the score of a token in a document
 // in general, the score is calculated as:
@@ -10,12 +11,74 @@ use std::collections::HashMap;
 pub trait Scorer: Send + Sync {
     fn query_weight(&self, token: &str) -> f32;
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32;
+
+    /// Finite upper bound for every non-negative value returned by
+    /// [`Self::doc_weight`]. Returning `None` disables score-independent
+    /// pruning where the posting format has no stored impact bounds.
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        None
+    }
+
+    /// Stable identity for the corpus-level inputs used by [`Self::doc_weight`].
+    ///
+    /// Implementations should return `Some` only when equal keys guarantee the
+    /// same document weight for every `(freq, doc_tokens)` pair. Scorers without
+    /// such an identity keep impact bounds in the query-local cache only.
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        None
+    }
+
+    /// The doc-length-dependent BM25 denominator addend, when `doc_weight`
+    /// factors as `(K1 + 1) * freq / (freq + addend)`; `None` for scorers
+    /// without that shape. Scoring hot loops use this to bake a per-norm-code
+    /// addend cache (Lucene's norm cache), which is bit-identical to calling
+    /// `doc_weight` because both paths evaluate the same expressions.
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        let _ = doc_tokens;
+        None
+    }
+}
+
+impl<T: Scorer + ?Sized> Scorer for Arc<T> {
+    fn query_weight(&self, token: &str) -> f32 {
+        self.as_ref().query_weight(token)
+    }
+
+    fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+        self.as_ref().doc_weight(freq, doc_tokens)
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        self.as_ref().doc_weight_upper_bound()
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        self.as_ref().doc_weight_cache_key()
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        self.as_ref().doc_norm(doc_tokens)
+    }
+}
+
+/// The frequency-dependent half of the BM25 doc weight; `doc_norm` is the
+/// doc-length addend (from [`Scorer::doc_norm`] or a per-norm-code cache).
+#[inline]
+pub(super) fn bm25_doc_weight_with_norm(freq: u32, doc_norm: f32) -> f32 {
+    let freq = freq as f32;
+    (K1 + 1.0) * freq / (freq + doc_norm)
 }
 
 // BM25 parameters
 pub const K1: f32 = 1.2;
 pub const B: f32 = 0.75;
 
+#[inline]
+fn bm25_doc_norm(doc_tokens: u32, avg_doc_length: f32) -> f32 {
+    let doc_tokens = doc_tokens as f32;
+    K1 * (1.0 - B + B * doc_tokens / avg_doc_length)
+}
+
 #[derive(Debug, Clone)]
 pub struct MemBM25Scorer {
     pub total_tokens: u64,
@@ -77,10 +140,20 @@ impl Scorer for MemBM25Scorer {
     }
 
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
-        let freq = freq as f32;
-        let doc_tokens = doc_tokens as f32;
-        let doc_norm = K1 * (1.0 - B + B * doc_tokens / self.avg_doc_length());
-        (K1 + 1.0) * freq / (freq + doc_norm)
+        let doc_norm = bm25_doc_norm(doc_tokens, self.avg_doc_length());
+        bm25_doc_weight_with_norm(freq, doc_norm)
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        Some(bm25_doc_norm(doc_tokens, self.avg_doc_length()))
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        Some(K1 + 1.0)
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        Some(u64::from(self.avg_doc_length().to_bits()))
     }
 }
 
@@ -141,10 +214,20 @@ impl Scorer for IndexBM25Scorer<'_> {
     }
 
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
-        let freq = freq as f32;
-        let doc_tokens = doc_tokens as f32;
-        let doc_norm = K1 * (1.0 - B + B * doc_tokens / self.avg_doc_length);
-        (K1 + 1.0) * freq / (freq + doc_norm)
+        let doc_norm = bm25_doc_norm(doc_tokens, self.avg_doc_length);
+        bm25_doc_weight_with_norm(freq, doc_norm)
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        Some(bm25_doc_norm(doc_tokens, self.avg_doc_length))
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        Some(K1 + 1.0)
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        Some(u64::from(self.avg_doc_length.to_bits()))
     }
 }
 
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 2c7a17465f8..5978621c462 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use lance_core::{Error, Result};
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize};
 use std::{env, path::PathBuf};
 
 #[cfg(feature = "tokenizer-jieba")]
@@ -23,27 +23,48 @@ use crate::scalar::inverted::tokenizer::document_tokenizer::{
     JsonTokenizer, LanceTokenizer, TextTokenizer,
 };
 use crate::scalar::inverted::{
-    InvertedListFormatVersion, default_fts_format_version, resolve_fts_format_version,
+    InvertedListFormatVersion, default_fts_format_version_for_block_size,
+    resolve_fts_format_version, validate_format_version_block_size,
 };
 pub use lance_tokenizer::Language;
 use lance_tokenizer::{
-    AsciiFoldingFilter, IcuTokenizer, LowerCaser, NgramTokenizer, RawTokenizer, RemoveLongFilter,
-    SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer, TextAnalyzerBuilder,
-    WhitespaceTokenizer,
+    AsciiFoldingFilter, CodeLexTokenizer, IcuTokenizer, LowerCaser, NgramTokenizer, RawTokenizer,
+    RemoveLongFilter, SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer, TextAnalyzerBuilder,
+    WhitespaceTokenizer, WordDelimiterFilter,
 };
 
+/// Posting block size for indexes whose metadata predates configurable block sizes.
+///
+/// This must remain 128 so that legacy on-disk data is decoded correctly.
+pub const LEGACY_BLOCK_SIZE: usize = 128;
+/// Default posting block size for newly created indexes when none is configured.
+///
+/// This intentionally matches [`LEGACY_BLOCK_SIZE`] today but may evolve independently.
+pub const DEFAULT_BLOCK_SIZE: usize = 128;
+pub const VALID_BLOCK_SIZES: [usize; 2] = [128, 256];
+const LANCE_FTS_FORMAT_VERSION_ENV_KEY: &str = "LANCE_FTS_FORMAT_VERSION";
+
 /// Tokenizer configs
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, PartialEq)]
 pub struct InvertedIndexParams {
-    /// lance tokenizer takes care of different data types, such as text, json, etc.
-    /// - 'text': parsing input documents into tokens
-    /// - 'json': parsing input json string into tokens
-    /// - none: auto type inference
+    /// Document-level tokenizer.
+    ///
+    /// This decides how Lance extracts searchable text from the stored value:
+    /// - `text`: plain string documents
+    /// - `json`: JSON string documents
+    /// - `None`: infer from the Arrow field type during index build
+    ///
+    /// The extracted text is then passed to `base_tokenizer`.
     pub(crate) lance_tokenizer: Option<String>,
-    /// base tokenizer:
+
+    /// Lexical tokenizer used after document-level text extraction.
+    ///
+    /// Client-facing analyzer profiles are resolved into this field and the
+    /// concrete filter options before params are persisted.
     /// - `simple`: splits tokens on whitespace and punctuation
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
+    /// - `code`: code-aware lexical tokenization
     /// - `icu`: ICU dictionary-based word segmentation
     /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
     /// - `lindera/*`: Lindera tokenizer
@@ -60,7 +81,6 @@ pub struct InvertedIndexParams {
     /// This can significantly increase the size of the index
     /// If false, only store the frequency of the term in the document
     /// Default is false
-    #[serde(default)]
     pub(crate) with_position: bool,
 
     /// maximum token length
@@ -69,15 +89,12 @@ pub struct InvertedIndexParams {
     pub(crate) max_token_length: Option<usize>,
 
     /// whether lower case tokens
-    #[serde(default = "bool_true")]
     pub(crate) lower_case: bool,
 
     /// whether apply stemming
-    #[serde(default = "bool_true")]
     pub(crate) stem: bool,
 
     /// whether remove stop words
-    #[serde(default = "bool_true")]
     pub(crate) remove_stop_words: bool,
 
     /// use customized stop words.
@@ -86,21 +103,36 @@ pub struct InvertedIndexParams {
     pub(crate) custom_stop_words: Option<Vec<String>>,
 
     /// ascii folding
-    #[serde(default = "bool_true")]
     pub(crate) ascii_folding: bool,
 
     /// min ngram length
-    #[serde(default = "default_min_ngram_length")]
     pub(crate) min_ngram_length: u32,
 
     /// max ngram length
-    #[serde(default = "default_max_ngram_length")]
     pub(crate) max_ngram_length: u32,
 
     /// whether prefix only
-    #[serde(default)]
     pub(crate) prefix_only: bool,
 
+    /// Number of documents in each compressed posting block.
+    ///
+    /// Missing serialized values come from indexes written before this
+    /// parameter existed and must read as 128 for backwards compatibility. New
+    /// indexes currently default to 128.
+    pub(crate) block_size: usize,
+
+    /// Split code identifiers into subwords.
+    pub(crate) split_identifiers: bool,
+
+    /// Split identifier subwords at letter/number boundaries.
+    pub(crate) split_on_numerics: bool,
+
+    /// Index a complete identifier in addition to subwords.
+    pub(crate) preserve_original: bool,
+
+    /// Index code operators such as `::`, `->`, and `!=`.
+    pub(crate) index_operators: bool,
+
     /// Total memory limit in MiB for the build stage.
     ///
     /// This is split evenly across FTS workers at build time. By default Lance
@@ -127,7 +159,10 @@ pub struct InvertedIndexParams {
     /// On-disk FTS format version to write when creating a new index.
     ///
     /// This is a build-time only parameter and is not persisted with the index.
-    /// If unset, Lance writes the current default FTS format.
+    /// If unset, new index creation falls back to
+    /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with
+    /// 128-document blocks writes v2, while code analysis or 256-document blocks
+    /// write v3.
     #[serde(
         rename = "format_version",
         skip_serializing,
@@ -137,6 +172,167 @@ pub struct InvertedIndexParams {
     pub(crate) format_version: Option<InvertedListFormatVersion>,
 }
 
+// Unknown fields must remain ignored because these params are persisted across Lance versions.
+#[derive(Debug, Deserialize)]
+struct RawInvertedIndexParams {
+    // Input-only preset expanded before constructing normalized params.
+    analyzer: Option<String>,
+    lance_tokenizer: Option<String>,
+    base_tokenizer: Option<String>,
+    language: Option<Language>,
+    with_position: Option<bool>,
+    #[serde(default, deserialize_with = "deserialize_explicit_option")]
+    max_token_length: Option<Option<usize>>,
+    lower_case: Option<bool>,
+    stem: Option<bool>,
+    remove_stop_words: Option<bool>,
+    custom_stop_words: Option<Vec<String>>,
+    ascii_folding: Option<bool>,
+    min_ngram_length: Option<u32>,
+    max_ngram_length: Option<u32>,
+    prefix_only: Option<bool>,
+    #[serde(default, deserialize_with = "deserialize_optional_block_size")]
+    block_size: Option<usize>,
+    split_identifiers: Option<bool>,
+    split_on_numerics: Option<bool>,
+    preserve_original: Option<bool>,
+    index_operators: Option<bool>,
+    #[serde(rename = "memory_limit", alias = "worker_memory_limit_mb")]
+    memory_limit_mb: Option<u64>,
+    #[serde(rename = "num_workers")]
+    num_workers: Option<usize>,
+    #[serde(
+        rename = "format_version",
+        default,
+        deserialize_with = "deserialize_format_version"
+    )]
+    format_version: Option<InvertedListFormatVersion>,
+}
+
+impl<'de> Deserialize<'de> for InvertedIndexParams {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        RawInvertedIndexParams::deserialize(deserializer)?
+            .resolve()
+            .map_err(serde::de::Error::custom)
+    }
+}
+
+impl RawInvertedIndexParams {
+    fn resolve(self) -> Result<InvertedIndexParams> {
+        let analyzer = match (
+            self.analyzer.as_deref().map(normalize_analyzer),
+            self.base_tokenizer.as_deref(),
+        ) {
+            (Some(Ok(analyzer)), _) => analyzer,
+            (Some(Err(err)), _) => return Err(err),
+            (None, Some("code")) => "code",
+            (None, _) => "text",
+        };
+
+        if analyzer == "text" && self.base_tokenizer.as_deref() == Some("code") {
+            return Err(Error::invalid_input(
+                "base_tokenizer='code' requires analyzer='code'".to_string(),
+            ));
+        }
+        if analyzer == "code"
+            && let Some(base_tokenizer) = self.base_tokenizer.as_deref()
+            && base_tokenizer != "code"
+        {
+            return Err(Error::invalid_input(format!(
+                "analyzer='code' requires base_tokenizer='code', got '{}'",
+                base_tokenizer
+            )));
+        }
+        if analyzer == "text"
+            && matches!(
+                (
+                    self.split_identifiers,
+                    self.split_on_numerics,
+                    self.preserve_original,
+                    self.index_operators,
+                ),
+                (Some(true), _, _, _)
+                    | (_, Some(true), _, _)
+                    | (_, _, Some(true), _)
+                    | (_, _, _, Some(true))
+            )
+        {
+            return Err(Error::invalid_input(
+                "code analyzer flags require analyzer='code'".to_string(),
+            ));
+        }
+
+        let mut params = match analyzer {
+            "code" => InvertedIndexParams::code(),
+            "text" => InvertedIndexParams::default(),
+            _ => unreachable!("analyzer is normalized above"),
+        };
+
+        if let Some(lance_tokenizer) = self.lance_tokenizer {
+            params.lance_tokenizer = Some(lance_tokenizer);
+        }
+        if let Some(base_tokenizer) = self.base_tokenizer {
+            params.base_tokenizer = base_tokenizer;
+        }
+        if let Some(language) = self.language {
+            params.language = language;
+        }
+        if let Some(with_position) = self.with_position {
+            params.with_position = with_position;
+        }
+        if let Some(max_token_length) = self.max_token_length {
+            params.max_token_length = max_token_length;
+        }
+        if let Some(lower_case) = self.lower_case {
+            params.lower_case = lower_case;
+        }
+        if let Some(stem) = self.stem {
+            params.stem = stem;
+        }
+        if let Some(remove_stop_words) = self.remove_stop_words {
+            params.remove_stop_words = remove_stop_words;
+        }
+        if let Some(custom_stop_words) = self.custom_stop_words {
+            params.custom_stop_words = Some(custom_stop_words);
+        }
+        if let Some(ascii_folding) = self.ascii_folding {
+            params.ascii_folding = ascii_folding;
+        }
+        if let Some(min_ngram_length) = self.min_ngram_length {
+            params.min_ngram_length = min_ngram_length;
+        }
+        if let Some(max_ngram_length) = self.max_ngram_length {
+            params.max_ngram_length = max_ngram_length;
+        }
+        if let Some(prefix_only) = self.prefix_only {
+            params.prefix_only = prefix_only;
+        }
+        if let Some(block_size) = self.block_size {
+            params.block_size = validate_block_size(block_size)?;
+        }
+        if let Some(split_identifiers) = self.split_identifiers {
+            params.split_identifiers = split_identifiers;
+        }
+        if let Some(split_on_numerics) = self.split_on_numerics {
+            params.split_on_numerics = split_on_numerics;
+        }
+        if let Some(preserve_original) = self.preserve_original {
+            params.preserve_original = preserve_original;
+        }
+        if let Some(index_operators) = self.index_operators {
+            params.index_operators = index_operators;
+        }
+        params.memory_limit_mb = self.memory_limit_mb;
+        params.num_workers = self.num_workers;
+        params.format_version = self.format_version;
+        params.validate()?;
+        Ok(params)
+    }
+}
+
 impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
     type Error = Error;
 
@@ -153,6 +349,15 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             min_ngram_length: params.min_ngram_length,
             max_ngram_length: params.max_ngram_length,
             prefix_only: params.prefix_only,
+            block_size: Some(params.block_size as u32),
+            code_config: (params.base_tokenizer == "code").then_some(
+                pbold::inverted_index_details::CodeTokenizerConfig {
+                    split_identifiers: params.split_identifiers,
+                    split_on_numerics: Some(params.split_on_numerics),
+                    preserve_original: Some(params.preserve_original),
+                    index_operators: params.index_operators,
+                },
+            ),
         })
     }
 }
@@ -161,34 +366,73 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
     type Error = Error;
 
     fn try_from(details: &pbold::InvertedIndexDetails) -> Result<Self> {
-        let defaults = Self::default();
-        Ok(Self {
-            lance_tokenizer: defaults.lance_tokenizer,
-            base_tokenizer: details
-                .base_tokenizer
-                .as_ref()
-                .cloned()
-                .unwrap_or_else(|| "simple".to_string()),
-            language: serde_json::from_str(details.language.as_str())?,
-            with_position: details.with_position,
-            max_token_length: details.max_token_length.map(|l| l as usize),
-            lower_case: details.lower_case,
-            stem: details.stem,
-            remove_stop_words: details.remove_stop_words,
-            custom_stop_words: defaults.custom_stop_words,
-            ascii_folding: details.ascii_folding,
-            min_ngram_length: details.min_ngram_length,
-            max_ngram_length: details.max_ngram_length,
-            prefix_only: details.prefix_only,
-            memory_limit_mb: defaults.memory_limit_mb,
-            num_workers: defaults.num_workers,
-            format_version: defaults.format_version,
-        })
+        if details.code_config.is_some() && details.base_tokenizer.as_deref() != Some("code") {
+            return Err(Error::invalid_input(
+                "code_config requires base_tokenizer='code'".to_string(),
+            ));
+        }
+        if details.base_tokenizer.is_none() && details.language.is_empty() {
+            let params = Self {
+                block_size: match details.block_size {
+                    Some(block_size) => validate_block_size(block_size as usize)?,
+                    None => LEGACY_BLOCK_SIZE,
+                },
+                ..Self::default()
+            };
+            return Ok(params);
+        }
+
+        let mut params = match details.base_tokenizer.as_deref() {
+            Some("code") => Self::code(),
+            _ => Self::default(),
+        };
+        params.base_tokenizer = details
+            .base_tokenizer
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| params.base_tokenizer.clone());
+        params.language = if details.language.is_empty() {
+            params.language
+        } else {
+            serde_json::from_str(details.language.as_str())?
+        };
+        params.with_position = details.with_position;
+        params.max_token_length = details.max_token_length.map(|l| l as usize);
+        params.lower_case = details.lower_case;
+        params.stem = details.stem;
+        params.remove_stop_words = details.remove_stop_words;
+        params.ascii_folding = details.ascii_folding;
+        params.min_ngram_length = details.min_ngram_length;
+        params.max_ngram_length = details.max_ngram_length;
+        params.prefix_only = details.prefix_only;
+        params.block_size = match details.block_size {
+            Some(block_size) => validate_block_size(block_size as usize)?,
+            None => LEGACY_BLOCK_SIZE,
+        };
+        if let Some(code_config) = &details.code_config {
+            params.split_identifiers = code_config.split_identifiers;
+            if let Some(split_on_numerics) = code_config.split_on_numerics {
+                params.split_on_numerics = split_on_numerics;
+            }
+            if let Some(preserve_original) = code_config.preserve_original {
+                params.preserve_original = preserve_original;
+            }
+            params.index_operators = code_config.index_operators;
+        }
+        params.validate()?;
+        Ok(params)
     }
 }
 
-fn bool_true() -> bool {
-    true
+fn normalize_analyzer(analyzer: &str) -> Result<&'static str> {
+    match analyzer {
+        "text" => Ok("text"),
+        "code" => Ok("code"),
+        other => Err(Error::invalid_input(format!(
+            "unknown analyzer '{}', expected 'text' or 'code'",
+            other
+        ))),
+    }
 }
 
 fn default_min_ngram_length() -> u32 {
@@ -199,6 +443,30 @@ fn default_max_ngram_length() -> u32 {
     3
 }
 
+fn invalid_block_size_message(block_size: usize) -> String {
+    format!("FTS inverted index block_size must be one of 128 or 256, got {block_size}")
+}
+
+pub fn validate_block_size(block_size: usize) -> Result<usize> {
+    if VALID_BLOCK_SIZES.contains(&block_size) {
+        Ok(block_size)
+    } else {
+        Err(Error::invalid_input(invalid_block_size_message(block_size)))
+    }
+}
+
+fn deserialize_optional_block_size<'de, D>(
+    deserializer: D,
+) -> std::result::Result<Option<usize>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Option::<usize>::deserialize(deserializer)?
+        .map(validate_block_size)
+        .transpose()
+        .map_err(serde::de::Error::custom)
+}
+
 fn deserialize_format_version<'de, D>(
     deserializer: D,
 ) -> std::result::Result<Option<InvertedListFormatVersion>, D::Error>
@@ -215,21 +483,52 @@ where
             .map(Some)
             .map_err(serde::de::Error::custom),
         serde_json::Value::Number(value) => {
-            let Some(value) = value.as_u64() else {
-                return Err(serde::de::Error::custom(
-                    "FTS format_version must be 1 or 2",
-                ));
+            let Some(format_version) = value.as_u64() else {
+                return Err(serde::de::Error::custom(format!(
+                    "FTS format_version must be 1, 2, or 3, got {value}"
+                )));
             };
-            resolve_fts_format_version(Some(&value.to_string()))
+            resolve_fts_format_version(Some(&format_version.to_string()))
                 .map(Some)
                 .map_err(serde::de::Error::custom)
         }
         other => Err(serde::de::Error::custom(format!(
-            "FTS format_version must be 1 or 2, got {other}"
+            "FTS format_version must be 1, 2, or 3, got {other}"
+        ))),
+    }
+}
+
+fn resolve_creation_format_version(
+    explicit: Option<InvertedListFormatVersion>,
+    default: InvertedListFormatVersion,
+) -> Result<InvertedListFormatVersion> {
+    if let Some(format_version) = explicit {
+        return Ok(format_version);
+    }
+
+    match env::var(LANCE_FTS_FORMAT_VERSION_ENV_KEY) {
+        Ok(value) => resolve_fts_format_version(Some(&value)).map_err(|err| {
+            Error::invalid_input(format!(
+                "invalid {LANCE_FTS_FORMAT_VERSION_ENV_KEY} value {value:?}: {err}"
+            ))
+        }),
+        Err(env::VarError::NotPresent) => Ok(default),
+        Err(env::VarError::NotUnicode(value)) => Err(Error::invalid_input(format!(
+            "invalid {LANCE_FTS_FORMAT_VERSION_ENV_KEY} value {value:?}: expected UTF-8 value 1, 2, or 3"
         ))),
     }
 }
 
+fn deserialize_explicit_option<'de, D, T>(
+    deserializer: D,
+) -> std::result::Result<Option<Option<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Deserialize<'de>,
+{
+    Option::<T>::deserialize(deserializer).map(Some)
+}
+
 impl Default for InvertedIndexParams {
     fn default() -> Self {
         Self::new("simple".to_owned(), Language::English)
@@ -243,6 +542,7 @@ impl InvertedIndexParams {
     /// - `simple`: splits tokens on whitespace and punctuation, default
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
+    /// - `code`: code identifier tokenization
     /// - `ngram`: N-Gram tokenizer
     /// - `icu`: ICU dictionary-based word segmentation
     /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
@@ -253,7 +553,7 @@ impl InvertedIndexParams {
     /// this is not used for `lindera/*` and `jieba/*` tokenizers.
     /// Default to `English`.
     pub fn new(base_tokenizer: String, language: Language) -> Self {
-        Self {
+        let mut params = Self {
             lance_tokenizer: None,
             base_tokenizer,
             language,
@@ -267,10 +567,82 @@ impl InvertedIndexParams {
             min_ngram_length: default_min_ngram_length(),
             max_ngram_length: default_max_ngram_length(),
             prefix_only: false,
+            block_size: DEFAULT_BLOCK_SIZE,
+            split_identifiers: false,
+            split_on_numerics: false,
+            preserve_original: false,
+            index_operators: false,
             memory_limit_mb: None,
             num_workers: None,
             format_version: None,
+        };
+        if params.base_tokenizer == "code" {
+            params.apply_code_defaults();
         }
+        params
+    }
+
+    fn apply_text_defaults(&mut self) {
+        self.base_tokenizer = "simple".to_string();
+        self.split_identifiers = false;
+        self.split_on_numerics = false;
+        self.preserve_original = false;
+        self.lower_case = true;
+        self.ascii_folding = true;
+        self.stem = true;
+        self.remove_stop_words = true;
+        self.index_operators = false;
+    }
+
+    fn apply_code_defaults(&mut self) {
+        self.base_tokenizer = "code".to_string();
+        self.split_identifiers = false;
+        self.split_on_numerics = true;
+        self.preserve_original = true;
+        self.lower_case = true;
+        self.ascii_folding = true;
+        self.stem = false;
+        self.remove_stop_words = false;
+        self.index_operators = false;
+    }
+
+    /// Create parameters for the code analyzer profile.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let tokenizer = InvertedIndexParams::code().build();
+    /// assert!(tokenizer.is_ok());
+    /// ```
+    pub fn code() -> Self {
+        Self::new("code".to_string(), Language::English)
+    }
+
+    /// Apply an analyzer profile to the concrete tokenizer parameters.
+    ///
+    /// The profile name is an input-time preset and is not persisted. Explicit
+    /// options applied after this method override the profile defaults.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::default()
+    ///     .analyzer("code")?
+    ///     .split_identifiers(true);
+    /// assert!(params.build().is_ok());
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
+    pub fn analyzer(mut self, analyzer: &str) -> Result<Self> {
+        match normalize_analyzer(analyzer)? {
+            "text" => self.apply_text_defaults(),
+            "code" => self.apply_code_defaults(),
+            _ => unreachable!("analyzer is normalized above"),
+        }
+        Ok(self)
     }
 
     pub fn lance_tokenizer(mut self, lance_tokenizer: String) -> Self {
@@ -278,8 +650,25 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set the lexical tokenizer implementation.
+    ///
+    /// Setting this to `"code"` selects the code analyzer defaults.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::default()
+    ///     .base_tokenizer("code".to_string())
+    ///     .split_identifiers(true);
+    /// assert!(params.build().is_ok());
+    /// ```
     pub fn base_tokenizer(mut self, base_tokenizer: String) -> Self {
         self.base_tokenizer = base_tokenizer;
+        if self.base_tokenizer == "code" {
+            self.apply_code_defaults();
+        }
         self
     }
 
@@ -335,6 +724,43 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set whether code identifiers are split into subwords.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::code()
+    ///     .split_identifiers(true)
+    ///     .split_on_numerics(false)
+    ///     .preserve_original(false)
+    ///     .index_operators(true);
+    /// assert!(params.build().is_ok());
+    /// ```
+    pub fn split_identifiers(mut self, split_identifiers: bool) -> Self {
+        self.split_identifiers = split_identifiers;
+        self
+    }
+
+    /// Set whether identifier subwords are split at letter/number boundaries.
+    pub fn split_on_numerics(mut self, split_on_numerics: bool) -> Self {
+        self.split_on_numerics = split_on_numerics;
+        self
+    }
+
+    /// Set whether the complete identifier is indexed alongside subwords.
+    pub fn preserve_original(mut self, preserve_original: bool) -> Self {
+        self.preserve_original = preserve_original;
+        self
+    }
+
+    /// Set whether operator tokens such as `::`, `->`, and `!=` are indexed.
+    pub fn index_operators(mut self, index_operators: bool) -> Self {
+        self.index_operators = index_operators;
+        self
+    }
+
     /// Set the minimum N-Gram length, only works when `base_tokenizer` is `ngram`.
     /// Must be greater than 0 and not greater than `max_ngram_length`.
     /// Default to 3.
@@ -358,6 +784,25 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set the compressed posting block size.
+    ///
+    /// Supported values are 128 and 256. Larger values reduce block-max metadata
+    /// and WAND skip granularity; smaller values preserve the legacy layout.
+    ///
+    /// `block_size = 256` is experimental and may introduce breaking changes.
+    /// Use `128` when stable compatibility with the legacy posting layout is required.
+    pub fn block_size(mut self, block_size: usize) -> Result<Self> {
+        self.block_size = validate_block_size(block_size)?;
+        Ok(self)
+    }
+
+    /// Get the compressed posting block size.
+    ///
+    /// `256` is experimental and may introduce breaking changes.
+    pub fn posting_block_size(&self) -> usize {
+        self.block_size
+    }
+
     pub fn memory_limit_mb(mut self, memory_limit_mb: u64) -> Self {
         self.memory_limit_mb = Some(memory_limit_mb);
         self
@@ -374,17 +819,41 @@ impl InvertedIndexParams {
 
     /// Set the on-disk FTS format version to use when creating a new index.
     ///
-    /// If unset, Lance writes the current default FTS format. Existing indexes
-    /// keep their own on-disk format during update and optimize operations.
+    /// If unset, new index creation falls back to
+    /// `LANCE_FTS_FORMAT_VERSION`. Without either override, text analysis with
+    /// 128-document blocks writes v2, while code analysis or 256-document blocks
+    /// write v3. Existing indexes keep their own format during update and
+    /// optimize operations.
     pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self {
         self.format_version = Some(format_version);
         self
     }
 
-    /// Resolve the requested FTS format version, falling back to Lance's default.
+    /// Resolve the requested FTS format version, falling back to the default for
+    /// the configured analyzer and block size.
     pub fn resolved_format_version(&self) -> InvertedListFormatVersion {
-        self.format_version
-            .unwrap_or_else(default_fts_format_version)
+        self.format_version.unwrap_or_else(|| {
+            if self.base_tokenizer == "code" {
+                InvertedListFormatVersion::V3
+            } else {
+                default_fts_format_version_for_block_size(self.block_size)
+                    .expect("InvertedIndexParams block_size must be validated before use")
+            }
+        })
+    }
+
+    /// Validate that the requested FTS format version can safely encode the
+    /// configured posting block size.
+    pub fn validate_format_version(&self) -> Result<()> {
+        let format_version = self.resolved_format_version();
+        validate_format_version_block_size(format_version, self.block_size)?;
+        if self.base_tokenizer == "code" && format_version != InvertedListFormatVersion::V3 {
+            return Err(Error::invalid_input(format!(
+                "base_tokenizer='code' requires FTS format_version=3, got {}",
+                format_version.index_version()
+            )));
+        }
+        Ok(())
     }
 
     /// Serialize params for the build/training path, including build-only fields.
@@ -414,7 +883,32 @@ impl InvertedIndexParams {
         Ok(value)
     }
 
+    /// Deserialize params for new index training, using the environment
+    /// override and current creation defaults for omitted fields.
+    pub(crate) fn from_training_json(params: &str) -> Result<Self> {
+        let supplied = serde_json::from_str::<serde_json::Value>(params)?;
+        let mut value = serde_json::to_value(Self::default())?;
+
+        let supplied = supplied.as_object().ok_or_else(|| {
+            Error::invalid_input("FTS inverted index params must be a JSON object".to_string())
+        })?;
+        let object = value
+            .as_object_mut()
+            .expect("inverted index params should serialize to a JSON object");
+        object.extend(supplied.clone());
+
+        let mut params: Self = serde_json::from_value(value)?;
+        let default_format_version = params.resolved_format_version();
+        params.format_version = Some(resolve_creation_format_version(
+            params.format_version,
+            default_format_version,
+        )?);
+        params.validate_format_version()?;
+        Ok(params)
+    }
+
     pub fn build(&self) -> Result<Box<dyn LanceTokenizer>> {
+        self.validate()?;
         let mut builder = self.build_base_tokenizer()?;
         if let Some(max_token_length) = self.max_token_length {
             builder = builder.filter_dynamic(RemoveLongFilter::limit(max_token_length));
@@ -459,11 +953,37 @@ impl InvertedIndexParams {
         }
     }
 
+    fn validate(&self) -> Result<()> {
+        validate_block_size(self.block_size)?;
+        if self.base_tokenizer != "code"
+            && (self.split_identifiers
+                || self.split_on_numerics
+                || self.preserve_original
+                || self.index_operators)
+        {
+            return Err(Error::invalid_input(
+                "code analyzer flags require base_tokenizer='code'".to_string(),
+            ));
+        }
+        Ok(())
+    }
+
     fn build_base_tokenizer(&self) -> Result<TextAnalyzerBuilder> {
         match self.base_tokenizer.as_str() {
             "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()),
             "whitespace" => Ok(TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()),
             "raw" => Ok(TextAnalyzer::builder(RawTokenizer::default()).dynamic()),
+            "code" => {
+                let mut builder =
+                    TextAnalyzer::builder(CodeLexTokenizer::new(self.index_operators)).dynamic();
+                if self.split_identifiers {
+                    builder = builder.filter_dynamic(WordDelimiterFilter::new(
+                        self.preserve_original,
+                        self.split_on_numerics,
+                    ));
+                }
+                Ok(builder)
+            }
             "icu" => Ok(TextAnalyzer::builder(IcuTokenizer::default()).dynamic()),
             "icu/split" => {
                 Ok(TextAnalyzer::builder(IcuTokenizer::default().with_simple_split()).dynamic())
@@ -519,9 +1039,13 @@ pub fn language_model_home() -> Option<PathBuf> {
 
 #[cfg(test)]
 mod tests {
-    use super::{InvertedIndexParams, InvertedListFormatVersion};
+    use crate::pbold;
+
+    use super::{InvertedIndexParams, InvertedListFormatVersion, Language};
+    use lance_core::Error;
     use lance_tokenizer::TokenStream;
     use rstest::rstest;
+    use serde_json::json;
 
     #[test]
     fn test_build_only_fields_are_not_serialized() {
@@ -580,6 +1104,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_training_json_preserves_disabled_max_token_length() {
+        let params = InvertedIndexParams::default().max_token_length(None);
+        let json = params.to_training_json().unwrap();
+        assert_eq!(json.get("max_token_length"), Some(&serde_json::Value::Null));
+
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.max_token_length, None);
+    }
+
     #[test]
     fn test_default_format_version_resolves_to_v2() {
         assert_eq!(
@@ -588,6 +1122,363 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_code_analyzer_resolves_defaults() {
+        let params: InvertedIndexParams = serde_json::from_value(json!({
+            "analyzer": "code"
+        }))
+        .unwrap();
+        assert_eq!(params.base_tokenizer, "code");
+        assert!(!params.split_identifiers);
+        assert!(params.split_on_numerics);
+        assert!(params.preserve_original);
+        assert!(params.lower_case);
+        assert!(params.ascii_folding);
+        assert!(!params.stem);
+        assert!(!params.remove_stop_words);
+        assert!(!params.index_operators);
+    }
+
+    #[test]
+    fn test_analyzer_profile_resolves_to_persisted_params() {
+        let from_profile: InvertedIndexParams = serde_json::from_value(json!({
+            "analyzer": "code",
+            "split_identifiers": true,
+            "preserve_original": false
+        }))
+        .unwrap();
+        let from_base_tokenizer: InvertedIndexParams = serde_json::from_value(json!({
+            "base_tokenizer": "code",
+            "split_identifiers": true,
+            "preserve_original": false
+        }))
+        .unwrap();
+        let from_constructor = InvertedIndexParams::new("code".to_string(), Language::English)
+            .split_identifiers(true)
+            .preserve_original(false);
+
+        assert_eq!(from_profile, from_base_tokenizer);
+        assert_eq!(from_profile, from_constructor);
+
+        let persisted = serde_json::to_value(&from_profile).unwrap();
+        assert!(persisted.get("analyzer").is_none());
+        assert_eq!(
+            serde_json::from_value::<InvertedIndexParams>(persisted).unwrap(),
+            from_profile
+        );
+    }
+
+    #[test]
+    fn test_block_size_256_defaults_to_v3() {
+        assert_eq!(
+            InvertedIndexParams::default()
+                .block_size(256)
+                .unwrap()
+                .resolved_format_version(),
+            InvertedListFormatVersion::V3
+        );
+    }
+
+    #[test]
+    fn test_code_analyzer_defaults_to_v3_for_supported_block_sizes() {
+        assert_eq!(
+            InvertedIndexParams::code().resolved_format_version(),
+            InvertedListFormatVersion::V3
+        );
+        assert_eq!(
+            InvertedIndexParams::code()
+                .block_size(256)
+                .unwrap()
+                .resolved_format_version(),
+            InvertedListFormatVersion::V3
+        );
+    }
+
+    #[test]
+    fn test_training_json_uses_v3_for_code_analyzer_and_supported_block_sizes() {
+        for block_size in [128, 256] {
+            let params = InvertedIndexParams::from_training_json(
+                &serde_json::json!({
+                    "base_tokenizer": "code",
+                    "block_size": block_size,
+                })
+                .to_string(),
+            )
+            .unwrap();
+            assert_eq!(params.format_version, Some(InvertedListFormatVersion::V3));
+        }
+    }
+
+    #[test]
+    fn test_text_analyzer_replaces_code_defaults() {
+        let params = InvertedIndexParams::code().analyzer("text").unwrap();
+        assert_eq!(params, InvertedIndexParams::default());
+    }
+
+    #[test]
+    fn test_code_analyzer_rejects_conflicting_base_tokenizer() {
+        let err = serde_json::from_value::<InvertedIndexParams>(json!({
+            "analyzer": "code",
+            "base_tokenizer": "simple"
+        }))
+        .unwrap_err();
+        assert!(err.to_string().contains("requires base_tokenizer='code'"));
+    }
+
+    #[test]
+    fn test_build_code_tokenizer_does_not_split_identifiers_by_default() {
+        let mut tokenizer = InvertedIndexParams::code().build().unwrap();
+        let mut stream = tokenizer.token_stream_for_doc(
+            "getUserName XMLHttpRequest parseHTML2JSON utf8_reader SCREAMING_SNAKE_CASE",
+        );
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| {
+            tokens.push((token.text.clone(), token.position, token.position_length))
+        });
+        assert_eq!(
+            tokens,
+            vec![
+                ("getusername".to_string(), 0, 1),
+                ("xmlhttprequest".to_string(), 1, 1),
+                ("parsehtml2json".to_string(), 2, 1),
+                ("utf8_reader".to_string(), 3, 1),
+                ("screaming_snake_case".to_string(), 4, 1),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_build_code_tokenizer_with_identifier_splitting() {
+        let mut tokenizer = InvertedIndexParams::code()
+            .split_identifiers(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_doc(
+            "getUserName XMLHttpRequest parseHTML2JSON utf8_reader SCREAMING_SNAKE_CASE",
+        );
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| {
+            tokens.push((token.text.clone(), token.position, token.position_length))
+        });
+        assert_eq!(
+            tokens,
+            vec![
+                ("getusername".to_string(), 0, 3),
+                ("get".to_string(), 0, 1),
+                ("user".to_string(), 1, 1),
+                ("name".to_string(), 2, 1),
+                ("xmlhttprequest".to_string(), 3, 3),
+                ("xml".to_string(), 3, 1),
+                ("http".to_string(), 4, 1),
+                ("request".to_string(), 5, 1),
+                ("parsehtml2json".to_string(), 6, 4),
+                ("parse".to_string(), 6, 1),
+                ("html".to_string(), 7, 1),
+                ("2".to_string(), 8, 1),
+                ("json".to_string(), 9, 1),
+                ("utf8_reader".to_string(), 10, 3),
+                ("utf".to_string(), 10, 1),
+                ("8".to_string(), 11, 1),
+                ("reader".to_string(), 12, 1),
+                ("screaming_snake_case".to_string(), 13, 3),
+                ("screaming".to_string(), 13, 1),
+                ("snake".to_string(), 14, 1),
+                ("case".to_string(), 15, 1),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_inverted_details_round_trip_code_params() {
+        let params = InvertedIndexParams::code()
+            .with_position(true)
+            .index_operators(true)
+            .preserve_original(false);
+        let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        let code_config = details.code_config.as_ref().unwrap();
+        assert_eq!(code_config.split_on_numerics, Some(true));
+        assert_eq!(code_config.preserve_original, Some(false));
+        let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
+        assert_eq!(round_tripped, params);
+    }
+
+    #[test]
+    fn test_inverted_details_uses_code_defaults_for_absent_flags() {
+        let mut details =
+            crate::pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::code()).unwrap();
+        let code_config = details.code_config.as_mut().unwrap();
+        code_config.split_on_numerics = None;
+        code_config.preserve_original = None;
+
+        let params = InvertedIndexParams::try_from(&details).unwrap();
+        assert!(params.split_on_numerics);
+        assert!(params.preserve_original);
+    }
+
+    #[test]
+    fn test_inverted_details_rejects_code_config_for_text_tokenizer() {
+        let mut details =
+            crate::pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::code()).unwrap();
+        details.base_tokenizer = Some("simple".to_string());
+
+        let err = InvertedIndexParams::try_from(&details).unwrap_err();
+        assert!(matches!(&err, Error::InvalidInput { .. }));
+        assert!(
+            err.to_string()
+                .contains("code_config requires base_tokenizer='code'")
+        );
+    }
+
+    #[test]
+    fn test_inverted_details_does_not_persist_document_tokenizer() {
+        let params = InvertedIndexParams::default().lance_tokenizer("json".to_string());
+        let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
+
+        assert_eq!(round_tripped.lance_tokenizer, None);
+    }
+
+    #[test]
+    fn test_format_version_must_match_block_size() {
+        InvertedIndexParams::default()
+            .format_version(InvertedListFormatVersion::V2)
+            .validate_format_version()
+            .unwrap();
+        InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .validate_format_version()
+            .unwrap();
+        InvertedIndexParams::default()
+            .format_version(InvertedListFormatVersion::V3)
+            .validate_format_version()
+            .unwrap();
+        InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .format_version(InvertedListFormatVersion::V3)
+            .validate_format_version()
+            .unwrap();
+
+        let err = InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .format_version(InvertedListFormatVersion::V2)
+            .validate_format_version()
+            .unwrap_err();
+        assert!(err.to_string().contains("block_size=256"));
+
+        let err = InvertedIndexParams::code()
+            .format_version(InvertedListFormatVersion::V2)
+            .validate_format_version()
+            .unwrap_err();
+        assert!(err.to_string().contains("requires FTS format_version=3"));
+    }
+
+    #[test]
+    fn test_training_json_rejects_incompatible_format_version_and_block_size() {
+        let err =
+            InvertedIndexParams::from_training_json(r#"{"block_size": 256, "format_version": 2}"#)
+                .unwrap_err();
+        assert!(err.to_string().contains("block_size=256"));
+    }
+
+    #[test]
+    fn test_training_json_invalid_numeric_format_version_includes_value() {
+        let err = InvertedIndexParams::from_training_json(r#"{"format_version": -1}"#).unwrap_err();
+        assert!(matches!(&err, lance_core::Error::Arrow { .. }));
+        assert!(err.to_string().contains("got -1"));
+    }
+
+    #[test]
+    fn test_training_json_ignores_unknown_fields() {
+        let params = InvertedIndexParams::from_training_json(
+            r#"{
+                "lower_case": false,
+                "skip_merge": true,
+                "future_parameter": {"enabled": true}
+            }"#,
+        )
+        .unwrap();
+
+        assert!(!params.lower_case);
+        let normalized = params.to_training_json().unwrap();
+        assert!(normalized.get("skip_merge").is_none());
+        assert!(normalized.get("future_parameter").is_none());
+    }
+
+    #[test]
+    fn test_block_size_default_serializes() {
+        let params = InvertedIndexParams::default();
+        assert_eq!(params.block_size, 128);
+        let json = serde_json::to_value(&params).unwrap();
+        assert_eq!(json.get("block_size"), Some(&serde_json::Value::from(128)));
+    }
+
+    #[test]
+    fn test_block_size_missing_metadata_falls_back_to_128() {
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        json.as_object_mut().unwrap().remove("block_size");
+
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.block_size, 128);
+    }
+
+    #[test]
+    fn test_block_size_details_conversion() {
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let details = pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        assert_eq!(details.block_size, Some(256));
+
+        let old_details = pbold::InvertedIndexDetails {
+            base_tokenizer: Some("simple".to_string()),
+            language: serde_json::to_string(&Language::English).unwrap(),
+            with_position: false,
+            max_token_length: Some(40),
+            lower_case: true,
+            stem: true,
+            remove_stop_words: true,
+            ascii_folding: true,
+            min_ngram_length: 3,
+            max_ngram_length: 3,
+            prefix_only: false,
+            block_size: None,
+            code_config: None,
+        };
+        let params = InvertedIndexParams::try_from(&old_details).unwrap();
+        assert_eq!(params.block_size, 128);
+    }
+
+    #[rstest]
+    #[case::block_size_128(128)]
+    #[case::block_size_256(256)]
+    fn test_block_size_accepts_supported_values(#[case] block_size: usize) {
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)
+            .unwrap();
+        assert_eq!(params.block_size, block_size);
+
+        let roundtrip: InvertedIndexParams =
+            serde_json::from_value(serde_json::to_value(&params).unwrap()).unwrap();
+        assert_eq!(roundtrip.block_size, block_size);
+    }
+
+    #[test]
+    fn test_block_size_rejects_invalid_values() {
+        let err = InvertedIndexParams::default().block_size(129).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
+
+        let err = InvertedIndexParams::default().block_size(512).unwrap_err();
+        assert!(err.to_string().contains("128 or 256"));
+
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        json.as_object_mut()
+            .unwrap()
+            .insert("block_size".to_string(), serde_json::Value::from(1024));
+        let err = serde_json::from_value::<InvertedIndexParams>(json).unwrap_err();
+        assert!(err.to_string().contains("128 or 256"));
+    }
+
     #[test]
     fn test_build_icu_tokenizer() {
         let mut tokenizer = InvertedIndexParams::default()
@@ -668,4 +1559,75 @@ mod tests {
         }
         assert_eq!(tokens, vec!["lance".to_string(), "data".to_string()]);
     }
+
+    // Common English pronouns/function words such as `you`/`my`/`your`/`we`
+    // must be removed by the ICU `all()` stop-word path. These are among the
+    // highest-frequency tokens, so leaking them builds pathologically large
+    // single-term posting lists (and previously overflowed the u32 posting-list
+    // size counter, panicking the whole index build). The leak is independent
+    // of stemming, so we assert it for both stem=false and stem=true.
+    #[rstest]
+    #[case::icu_no_stem("icu", false)]
+    #[case::icu_stem("icu", true)]
+    #[case::icu_split_no_stem("icu/split", false)]
+    #[case::icu_split_stem("icu/split", true)]
+    // `simple` is the recommended tokenizer for monolingual English corpora and
+    // uses StopWordFilter::new(English) rather than the ICU all() path, so it
+    // must be covered too.
+    #[case::simple_no_stem("simple", false)]
+    #[case::simple_stem("simple", true)]
+    fn test_icu_common_english_stop_words_do_not_leak(
+        #[case] base_tokenizer: &str,
+        #[case] stem: bool,
+    ) {
+        let mut tokenizer = InvertedIndexParams::default()
+            .base_tokenizer(base_tokenizer.to_string())
+            .stem(stem)
+            .remove_stop_words(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("you my your we lance data");
+        let tokens: Vec<String> = std::iter::from_fn(|| stream.next().map(|t| t.text.clone()))
+            .filter(|t| matches!(t.as_str(), "you" | "my" | "your" | "we"))
+            .collect();
+        assert!(
+            tokens.is_empty(),
+            "common English stop words leaked through the icu pipeline (stem={stem}): {tokens:?}"
+        );
+    }
+
+    // Common Chinese function words/particles (了 是 在 的 和 有 我) are the
+    // highest-frequency Chinese tokens; like the English pronouns they must be
+    // removed by the ICU `all()` stop-word path so they don't build huge
+    // posting lists. Real content words (英语 = "English", 数据 = "data") must
+    // survive. ICU dictionary segmentation splits the input into words, so this
+    // exercises the CJK stop-word path end to end.
+    #[rstest]
+    #[case::icu("icu")]
+    #[case::icu_split("icu/split")]
+    fn test_icu_common_chinese_stop_words_do_not_leak(#[case] base_tokenizer: &str) {
+        let mut tokenizer = InvertedIndexParams::default()
+            .base_tokenizer(base_tokenizer.to_string())
+            .stem(true)
+            .remove_stop_words(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("我 在 有 了 是 的 和 英语 数据");
+        let tokens: Vec<String> =
+            std::iter::from_fn(|| stream.next().map(|t| t.text.clone())).collect();
+        let stop = ["我", "在", "有", "了", "是", "的", "和"];
+        let leaked: Vec<&String> = tokens
+            .iter()
+            .filter(|t| stop.contains(&t.as_str()))
+            .collect();
+        assert!(
+            leaked.is_empty(),
+            "common Chinese stop words leaked through the icu pipeline: {leaked:?} (all tokens: {tokens:?})"
+        );
+        // The real content words must still be indexed.
+        assert!(
+            tokens.iter().any(|t| t == "英语"),
+            "content word 英语 was dropped: {tokens:?}"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs
index 0fc8b95cddb..f730d6f9d4b 100644
--- a/rust/lance-index/src/scalar/inverted/wand.rs
+++ b/rust/lance-index/src/scalar/inverted/wand.rs
@@ -1,11 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::ops::Deref;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, LazyLock};
 use std::{
-    cell::UnsafeCell,
+    cell::{RefCell, UnsafeCell},
     collections::{BinaryHeap, VecDeque},
 };
 use std::{cmp::Reverse, fmt::Debug};
@@ -14,23 +13,25 @@ use arrow::array::AsArray;
 use arrow::datatypes::Int32Type;
 use arrow_array::Array;
 use itertools::Itertools;
-use lance_core::Result;
 use lance_core::utils::address::RowAddress;
+use lance_core::{Error, Result};
 use lance_select::RowAddrMask;
 
 use crate::metrics::MetricsCollector;
 
 use super::{
     CompressedPositionStorage,
+    impact::{IMPACT_LEVEL1_BLOCKS, ImpactScoreCache, ImpactSkipData},
+    index::{PositionStreamCodec, dequantize_doc_length},
     query::Operator,
-    scorer::{K1, idf},
+    scorer::{K1, bm25_doc_weight_with_norm, idf},
 };
 use super::{
     CompressedPostingList, DocSet, PostingList, RawDocInfo,
     builder::ScoredDoc,
     encoding::{
-        decode_position_stream_block, decompress_positions, decompress_posting_block,
-        decompress_posting_remainder,
+        MAX_POSTING_BLOCK_SIZE, decode_position_stream_block, decompress_positions,
+        decompress_posting_block, decompress_posting_remainder, seek_packed_doc_positions,
     },
     query::FtsSearchParams,
     scorer::Scorer,
@@ -38,6 +39,163 @@ use super::{
 use super::{DocInfo, builder::BLOCK_SIZE};
 
 const TERMINATED_DOC_ID: u64 = u64::MAX;
+
+/// Top-k heap entry: (scored doc, doc length, posting doc id, frequency slot).
+/// The (term, freq) pairs live outside the heap so heap churn moves a compact
+/// tuple instead of a `Vec`.
+type TopKHeap = BinaryHeap<Reverse<(ScoredDoc, u32, u64, u32)>>;
+
+/// Reusable (term, freq) storage for live heap candidates. Replacing the k-th
+/// result reuses its slot and retained `Vec` capacity, so memory stays bounded
+/// by the live top-k rather than every historical heap admission.
+struct FrequencySlots {
+    slots: Vec<Vec<(u32, u32)>>,
+}
+
+impl FrequencySlots {
+    fn with_capacity(capacity: usize) -> Self {
+        Self {
+            slots: Vec::with_capacity(capacity),
+        }
+    }
+
+    fn push(&mut self, pairs: impl Iterator<Item = (u32, u32)>) -> Result<u32> {
+        let slot = u32::try_from(self.slots.len()).map_err(|_| {
+            Error::internal(format!(
+                "FTS top-k frequency slot count {} exceeds u32::MAX",
+                self.slots.len()
+            ))
+        })?;
+        self.slots.push(pairs.collect());
+        Ok(slot)
+    }
+
+    fn replace(&mut self, slot: u32, pairs: impl Iterator<Item = (u32, u32)>) -> Result<()> {
+        let num_slots = self.slots.len();
+        let slot = self.slots.get_mut(slot as usize).ok_or_else(|| {
+            Error::internal(format!(
+                "FTS top-k frequency slot {slot} is out of bounds for {num_slots} slots"
+            ))
+        })?;
+        slot.clear();
+        slot.extend(pairs);
+        Ok(())
+    }
+
+    fn take(&mut self, slot: u32) -> Result<Vec<(u32, u32)>> {
+        let num_slots = self.slots.len();
+        self.slots
+            .get_mut(slot as usize)
+            .map(std::mem::take)
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "FTS top-k frequency slot {slot} is out of bounds for {num_slots} slots"
+                ))
+            })
+    }
+}
+
+/// Owns the top-k heap and the frequency slots referenced by its entries.
+struct TopKCollector {
+    limit: usize,
+    heap: TopKHeap,
+    frequency_slots: FrequencySlots,
+}
+
+impl TopKCollector {
+    fn new(limit: usize, initial_capacity: usize) -> Self {
+        let initial_capacity = initial_capacity.min(limit);
+        Self {
+            limit,
+            heap: BinaryHeap::with_capacity(initial_capacity),
+            frequency_slots: FrequencySlots::with_capacity(initial_capacity),
+        }
+    }
+
+    /// Insert a competitive result. When the heap is full, its evicted entry's
+    /// frequency slot is cleared and reused before the replacement is pushed.
+    fn insert(
+        &mut self,
+        doc: ScoredDoc,
+        doc_length: u32,
+        posting_doc_id: u64,
+        pairs: impl Iterator<Item = (u32, u32)>,
+    ) -> Result<bool> {
+        if self.limit == 0 {
+            return Ok(false);
+        }
+        if self.heap.len() > self.limit {
+            return Err(Error::internal(format!(
+                "FTS top-k heap length {} exceeds limit {}",
+                self.heap.len(),
+                self.limit
+            )));
+        }
+
+        let frequency_slot = if self.heap.len() == self.limit {
+            let Some(kth_score) = self.heap.peek().map(|entry| entry.0.0.score.0) else {
+                return Err(Error::internal(
+                    "FTS top-k heap is empty while its nonzero limit is reached",
+                ));
+            };
+            // Preserve the existing collector semantics for non-finite custom
+            // scorer output: only a strictly greater raw f32 replaces k-th.
+            if doc.score.0.partial_cmp(&kth_score) != Some(std::cmp::Ordering::Greater) {
+                return Ok(false);
+            }
+            let Some(Reverse((_, _, _, frequency_slot))) = self.heap.pop() else {
+                return Err(Error::internal(
+                    "FTS top-k heap entry disappeared during replacement",
+                ));
+            };
+            self.frequency_slots.replace(frequency_slot, pairs)?;
+            frequency_slot
+        } else {
+            self.frequency_slots.push(pairs)?
+        };
+
+        self.heap
+            .push(Reverse((doc, doc_length, posting_doc_id, frequency_slot)));
+        Ok(true)
+    }
+
+    fn kth_score_if_full(&self) -> Option<f32> {
+        if self.heap.len() == self.limit {
+            self.heap.peek().map(|entry| entry.0.0.score.0)
+        } else {
+            None
+        }
+    }
+
+    fn into_candidates(
+        self,
+        mut to_addr: impl FnMut(u64) -> CandidateAddr,
+    ) -> Result<Vec<DocCandidate>> {
+        let Self {
+            heap,
+            mut frequency_slots,
+            ..
+        } = self;
+        heap.into_iter()
+            .map(
+                |Reverse((doc, doc_length, posting_doc_id, frequency_slot))| {
+                    Ok(DocCandidate {
+                        addr: to_addr(doc.row_id),
+                        posting_doc_id,
+                        freqs: frequency_slots.take(frequency_slot)?,
+                        doc_length,
+                    })
+                },
+            )
+            .collect()
+    }
+
+    #[cfg(test)]
+    fn num_frequency_slots(&self) -> usize {
+        self.frequency_slots.slots.len()
+    }
+}
+const LINEAR_BLOCK_SKIP_LIMIT: usize = 8;
 pub static FLAT_SEARCH_PERCENT_THRESHOLD: LazyLock<u64> = LazyLock::new(|| {
     std::env::var("LANCE_FLAT_SEARCH_PERCENT_THRESHOLD")
         .unwrap_or_else(|_| "10".to_string())
@@ -45,6 +203,189 @@ pub static FLAT_SEARCH_PERCENT_THRESHOLD: LazyLock<u64> = LazyLock::new(|| {
         .unwrap_or(10)
 });
 
+/// Single shared definition of the flat-search predicate: [`Wand::search`]
+/// and `LazyDocSet::docs_for_wand` MUST agree — a masked query scored
+/// without row_ids silently skips the `mask.selected` filter.
+pub(super) fn should_flat_search(operator: Operator, mask: &RowAddrMask, num_docs: u64) -> bool {
+    operator == Operator::Or
+        && mask.iter_addrs().is_some()
+        && mask.max_len().is_some_and(|num_rows_matched| {
+            num_rows_matched * 100 <= *FLAT_SEARCH_PERCENT_THRESHOLD * num_docs
+        })
+}
+// Bulk MAXSCORE path for top-k disjunctions (Lucene MaxScoreBulkScorer
+// style). Default on: with right-sized partitions it wins by a wide margin
+// (Lucene-parity latency) and its results are score-identical to the classic
+// WAND loop. LANCE_FTS_MAXSCORE=0 opts back into the classic loop.
+static USE_MAXSCORE_SEARCH: LazyLock<bool> =
+    LazyLock::new(|| std::env::var("LANCE_FTS_MAXSCORE").as_deref() != Ok("0"));
+// Bulk conjunction path for top-k AND / phrase queries: block-max window
+// skipping plus a slice-level merge over decompressed blocks, replacing the
+// per-doc `next()` leapfrog. Results are identical to the classic AND loop.
+// LANCE_FTS_BULK_AND accepts auto (default), on/1, or off/0. Auto enables the
+// bulk path only for its consistently faster two- and three-clause kernels.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+enum BulkAndMode {
+    #[default]
+    Auto,
+    On,
+    Off,
+}
+
+impl BulkAndMode {
+    fn parse(value: &str) -> Option<Self> {
+        let value = value.trim();
+        if value.eq_ignore_ascii_case("auto") {
+            Some(Self::Auto)
+        } else if value.eq_ignore_ascii_case("on") || value == "1" {
+            Some(Self::On)
+        } else if value.eq_ignore_ascii_case("off") || value == "0" {
+            Some(Self::Off)
+        } else {
+            None
+        }
+    }
+
+    const fn enabled_for(self, num_clauses: usize) -> bool {
+        match self {
+            Self::Auto => matches!(num_clauses, 2 | 3),
+            Self::On => true,
+            Self::Off => false,
+        }
+    }
+}
+
+fn bulk_and_mode_from_env() -> BulkAndMode {
+    match std::env::var("LANCE_FTS_BULK_AND") {
+        Ok(value) => BulkAndMode::parse(&value).unwrap_or_else(|| {
+            log::warn!(
+                "Invalid LANCE_FTS_BULK_AND value {value:?}; expected auto, on/1, or off/0; \
+                 falling back to auto"
+            );
+            BulkAndMode::Auto
+        }),
+        Err(std::env::VarError::NotPresent) => BulkAndMode::Auto,
+        Err(std::env::VarError::NotUnicode(value)) => {
+            log::warn!(
+                "Invalid non-Unicode LANCE_FTS_BULK_AND value {value:?}; expected auto, on/1, \
+                 or off/0; falling back to auto"
+            );
+            BulkAndMode::Auto
+        }
+    }
+}
+
+static BULK_AND_MODE: LazyLock<BulkAndMode> = LazyLock::new(bulk_and_mode_from_env);
+
+#[cfg(target_arch = "x86_64")]
+static HAS_AVX2: LazyLock<bool> = LazyLock::new(|| std::arch::is_x86_feature_detected!("avx2"));
+
+/// First index in `[pos, end)` where `docs[index] >= target` (scalar).
+/// Posting-block doc ids stay below 2^31, which the AVX2 variant relies on.
+#[inline]
+unsafe fn find_next_geq_scalar(docs: *const u32, mut pos: usize, end: usize, target: u32) -> usize {
+    unsafe {
+        while pos < end && *docs.add(pos) < target {
+            pos += 1;
+        }
+    }
+    pos
+}
+
+/// AVX2 `find_next_geq` (the analogue of Lucene's VectorUtil.findNextGEQ):
+/// branchless 8-wide compare+movemask kills the mispredicted exits that
+/// dominate the scalar catch-up scan on irregular doc gaps.
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn find_next_geq_avx2(docs: *const u32, mut pos: usize, end: usize, target: u32) -> usize {
+    use core::arch::x86_64::*;
+    debug_assert!(target <= i32::MAX as u32);
+    unsafe {
+        let target_lanes = _mm256_set1_epi32(target as i32);
+        while pos + 8 <= end {
+            let docs_lanes = _mm256_loadu_si256(docs.add(pos) as *const __m256i);
+            // lane mask of docs[i] < target; doc ids < 2^31 keep the signed
+            // compare equivalent to unsigned.
+            let below = _mm256_cmpgt_epi32(target_lanes, docs_lanes);
+            let mask = _mm256_movemask_ps(_mm256_castsi256_ps(below)) as u32;
+            if mask != 0xFF {
+                return pos + mask.trailing_ones() as usize;
+            }
+            pos += 8;
+        }
+        find_next_geq_scalar(docs, pos, end, target)
+    }
+}
+
+#[inline]
+unsafe fn find_next_geq(docs: *const u32, pos: usize, end: usize, target: u32) -> usize {
+    #[cfg(target_arch = "x86_64")]
+    if *HAS_AVX2 {
+        return unsafe { find_next_geq_avx2(docs, pos, end, target) };
+    }
+    unsafe { find_next_geq_scalar(docs, pos, end, target) }
+}
+
+#[inline]
+fn conservative_bm25_upper_bound(query_weight: f32) -> f32 {
+    if query_weight <= 0.0 {
+        0.0
+    } else {
+        query_weight * (K1 + 1.0)
+    }
+}
+
+#[inline]
+fn scorer_upper_bound<S: Scorer + ?Sized>(query_weight: f32, scorer: &S) -> f32 {
+    if query_weight.is_nan() {
+        return f32::INFINITY;
+    }
+    if query_weight <= 0.0 {
+        return 0.0;
+    }
+    match scorer.doc_weight_upper_bound() {
+        Some(bound) if bound.is_finite() && bound >= 0.0 => query_weight * bound,
+        _ => f32::INFINITY,
+    }
+}
+
+/// Per-term data retained after same-position postings are unioned.
+#[derive(Debug)]
+pub(super) struct GroupedTermScorer {
+    query_weight: f32,
+    freqs_by_posting_doc_id: Vec<(u64, u32)>,
+}
+
+impl GroupedTermScorer {
+    pub(super) fn new(query_weight: f32, posting: &PostingList) -> Self {
+        let freqs_by_posting_doc_id = posting
+            .iter()
+            .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq))
+            .collect();
+        Self {
+            query_weight,
+            freqs_by_posting_doc_id,
+        }
+    }
+
+    pub(super) fn query_weight(&self) -> f32 {
+        self.query_weight
+    }
+
+    pub(super) fn frequency(&self, posting_doc_id: u64) -> Option<u32> {
+        self.freqs_by_posting_doc_id
+            .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id)
+            .ok()
+            .map(|index| self.freqs_by_posting_doc_id[index].1)
+    }
+
+    fn score<S: Scorer + ?Sized>(&self, posting_doc_id: u64, doc_length: u32, scorer: &S) -> f32 {
+        self.frequency(posting_doc_id)
+            .map(|freq| self.query_weight * scorer.doc_weight(freq, doc_length))
+            .unwrap_or_default()
+    }
+}
+
 pub struct PostingIterator {
     token: String,
     token_id: u32,
@@ -55,7 +396,18 @@ pub struct PostingIterator {
     index: usize,
     // the index of current block, this can be changed by `next() and shallow_next()`
     block_idx: usize,
+    current_doc: Option<DocInfo>,
     approximate_upper_bound: f32,
+    // Stored block maxima may use partition-local statistics. Queries scored
+    // with corpus-wide statistics must use a scorer-provided bound instead.
+    use_scorer_upper_bound: bool,
+    // The union posting drives iteration and pruning; these terms produce the
+    // exact score for each candidate document.
+    grouped_terms: Option<Arc<[GroupedTermScorer]>>,
+    // Position cursors temporarily own this buffer and return it on drop. This
+    // keeps repeated cursor creation allocation-free without lending a slice
+    // out of the interior-mutable compressed state.
+    position_scratch: RefCell<Option<Vec<u32>>>,
 
     // for compressed posting list
     compressed: Option<UnsafeCell<CompressedState>>,
@@ -66,24 +418,46 @@ struct CompressedState {
     block_idx: usize,
     doc_ids: Vec<u32>,
     freqs: Vec<u32>,
-    buffer: Box<[u32; BLOCK_SIZE]>,
+    buffer: Box<[u32; MAX_POSTING_BLOCK_SIZE]>,
     position_block_idx: Option<usize>,
     position_values: Vec<u32>,
     position_offsets: Vec<usize>,
+    // Seek state for PackedDelta position blocks: the lazily-built group
+    // header index, the last unpacked group (memoized), the decoded varint
+    // tail, and the block's total delta count. Together these let a phrase
+    // check decode just the candidate doc's positions instead of the whole
+    // 256-doc position block.
+    position_group_offsets: Vec<usize>,
+    position_unpacked_group: Box<[u32; BLOCK_SIZE]>,
+    position_unpacked_group_idx: Option<usize>,
+    position_tail: Vec<u32>,
+    position_total_deltas: usize,
     block_max_window: BlockMaxWindow,
+    // Lucene-style anchored impact score caches: one slot per level, keyed by
+    // the entry the block cursor currently sits in. Each holds
+    // (entry_idx, doc_up_to, max_score). See `impact_level0`/`impact_level1`.
+    level0_cache: Option<(usize, u32, f32)>,
+    level1_cache: Option<(usize, u32, f32)>,
 }
 
 impl CompressedState {
-    fn new() -> Self {
+    fn new(block_size: usize) -> Self {
         Self {
             block_idx: 0,
-            doc_ids: Vec::with_capacity(BLOCK_SIZE),
-            freqs: Vec::with_capacity(BLOCK_SIZE),
-            buffer: Box::new([0; BLOCK_SIZE]),
+            doc_ids: Vec::with_capacity(block_size),
+            freqs: Vec::with_capacity(block_size),
+            buffer: Box::new([0; MAX_POSTING_BLOCK_SIZE]),
             position_block_idx: None,
             position_values: Vec::new(),
             position_offsets: Vec::new(),
+            position_group_offsets: Vec::new(),
+            position_unpacked_group: Box::new([0; BLOCK_SIZE]),
+            position_unpacked_group_idx: None,
+            position_tail: Vec::new(),
+            position_total_deltas: 0,
             block_max_window: BlockMaxWindow::new(),
+            level0_cache: None,
+            level1_cache: None,
         }
     }
 
@@ -95,21 +469,29 @@ impl CompressedState {
         num_blocks: usize,
         length: u32,
         tail_codec: super::PostingTailCodec,
+        block_size: usize,
     ) {
         self.doc_ids.clear();
         self.freqs.clear();
 
-        let remainder = length as usize % BLOCK_SIZE;
+        let remainder = length as usize % block_size;
         if block_idx + 1 == num_blocks && remainder != 0 {
             decompress_posting_remainder(
                 block,
                 remainder,
                 tail_codec,
+                block_size,
                 &mut self.doc_ids,
                 &mut self.freqs,
             );
         } else {
-            decompress_posting_block(block, &mut self.buffer, &mut self.doc_ids, &mut self.freqs);
+            decompress_posting_block(
+                block,
+                &mut self.buffer[..],
+                &mut self.doc_ids,
+                &mut self.freqs,
+                block_size,
+            );
         }
         self.block_idx = block_idx;
         self.position_block_idx = None;
@@ -122,9 +504,12 @@ impl CompressedState {
 struct BlockMaxWindow {
     // Sliding block range used for Lucene-style getMaxScore(upTo). The deque is
     // monotonic by score and covers blocks in [start_block_idx, next_block_idx).
+    // Only used for compressed lists without impact skip data; impact lists
+    // answer window max scores from the anchored level caches instead.
     start_block_idx: usize,
     next_block_idx: usize,
     max_scores: VecDeque<(usize, f32)>,
+    impact_score_cache: ImpactScoreCache,
 }
 
 struct BlockMaxScore {
@@ -138,6 +523,7 @@ impl BlockMaxWindow {
             start_block_idx: 0,
             next_block_idx: 0,
             max_scores: VecDeque::new(),
+            impact_score_cache: ImpactScoreCache::default(),
         }
     }
 
@@ -147,11 +533,13 @@ impl BlockMaxWindow {
         self.max_scores.clear();
     }
 
-    fn max_score_up_to(
+    fn max_score_up_to<S: Scorer + ?Sized>(
         &mut self,
         list: &CompressedPostingList,
         start_block_idx: usize,
         up_to: u64,
+        query_weight: f32,
+        scorer: &S,
     ) -> BlockMaxScore {
         if start_block_idx >= list.blocks.len() {
             self.reset(start_block_idx);
@@ -177,12 +565,31 @@ impl BlockMaxWindow {
             };
         }
 
+        // 256-document-block postings score quantized document lengths, which can be shorter than
+        // the exact lengths used to bake a legacy max score. Without impacts,
+        // use the score-independent BM25 ceiling instead of that stale bound.
+        if list.block_size == MAX_POSTING_BLOCK_SIZE {
+            self.reset(start_block_idx);
+            return BlockMaxScore {
+                score: scorer_upper_bound(query_weight, scorer),
+                blocks_scanned: 0,
+            };
+        }
+
         self.next_block_idx = self.next_block_idx.max(start_block_idx);
         let mut blocks_scanned = 0;
         while self.next_block_idx < list.blocks.len()
             && list.block_least_doc_id(self.next_block_idx) as u64 <= up_to
         {
-            let score = list.block_max_score(self.next_block_idx);
+            let score = match list.impacts.as_ref() {
+                Some(impacts) => impacts.level0_score_cached(
+                    self.next_block_idx,
+                    query_weight,
+                    scorer,
+                    &mut self.impact_score_cache,
+                ),
+                None => list.block_max_score(self.next_block_idx),
+            };
             while matches!(self.max_scores.back(), Some((_, old_score)) if *old_score <= score) {
                 self.max_scores.pop_back();
             }
@@ -256,6 +663,104 @@ impl Ord for PostingIterator {
 }
 
 impl PostingIterator {
+    fn block_idx_for_doc(
+        &self,
+        list: &CompressedPostingList,
+        mut block_idx: usize,
+        least_id: u32,
+    ) -> usize {
+        let mut linear_skips = 0;
+        while block_idx + 1 < list.blocks.len() && linear_skips < LINEAR_BLOCK_SKIP_LIMIT {
+            if list.block_least_doc_id(block_idx + 1) > least_id {
+                return block_idx;
+            }
+            block_idx += 1;
+            linear_skips += 1;
+        }
+
+        if block_idx + 1 >= list.blocks.len() {
+            return block_idx;
+        }
+
+        if let Some(impacts) = list.impacts.as_ref()
+            && let Some(block_idx) =
+                self.block_idx_for_doc_with_impacts(list, impacts, block_idx, least_id)
+        {
+            return block_idx;
+        }
+
+        self.block_idx_for_doc_by_least_doc_id(list, block_idx, least_id, list.blocks.len())
+    }
+
+    fn block_idx_for_doc_with_impacts(
+        &self,
+        list: &CompressedPostingList,
+        impacts: &ImpactSkipData,
+        mut block_idx: usize,
+        least_id: u32,
+    ) -> Option<usize> {
+        while block_idx + 1 < list.blocks.len() {
+            let group_idx = (block_idx + 1) / IMPACT_LEVEL1_BLOCKS;
+            let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(list.blocks.len());
+            let group_doc_up_to = impacts.level1_doc_up_to(group_idx)?;
+            if group_doc_up_to < least_id {
+                block_idx = group_end - 1;
+                continue;
+            }
+            if group_doc_up_to == least_id {
+                return Some(group_end - 1);
+            }
+            return Some(
+                self.block_idx_for_doc_by_least_doc_id(list, block_idx, least_id, group_end),
+            );
+        }
+        Some(block_idx)
+    }
+
+    fn block_idx_for_doc_by_least_doc_id(
+        &self,
+        list: &CompressedPostingList,
+        block_idx: usize,
+        least_id: u32,
+        right: usize,
+    ) -> usize {
+        let mut left = block_idx + 1;
+        let mut right = right;
+        while left < right {
+            let mid = left + (right - left) / 2;
+            if list.block_least_doc_id(mid) <= least_id {
+                left = mid + 1;
+            } else {
+                right = mid;
+            }
+        }
+        left - 1
+    }
+
+    #[inline]
+    fn block_end_doc(&self) -> u64 {
+        self.next_block_first_doc()
+            .map(|doc| doc.saturating_sub(1))
+            .unwrap_or(TERMINATED_DOC_ID)
+    }
+
+    /// Level1 bound of the group holding the current block, for group-wide
+    /// skipping: (group doc_up_to, group max score). `None` when the list has
+    /// no impact skip data or the group entry is missing/malformed.
+    fn impact_group_bound<S: Scorer + ?Sized>(&self, scorer: &S) -> Option<(u64, f32)> {
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                let impacts = list.impacts.as_ref()?;
+                let (doc_up_to, score) = self.impact_level1(impacts, scorer);
+                if doc_up_to == u32::MAX {
+                    return None;
+                }
+                Some((u64::from(doc_up_to), score))
+            }
+            PostingList::Plain(_) => None,
+        }
+    }
+
     #[inline]
     fn compressed_state_ptr(&self) -> *mut CompressedState {
         debug_assert!(self.compressed.is_some());
@@ -279,6 +784,7 @@ impl PostingIterator {
                 list.blocks.len(),
                 list.length,
                 list.posting_tail_codec,
+                list.block_size,
             );
         }
         compressed as *mut CompressedState
@@ -303,14 +809,29 @@ impl PostingIterator {
         list: PostingList,
         num_doc: usize,
     ) -> Self {
-        let approximate_upper_bound = match list.max_score() {
-            Some(max_score) => max_score,
-            None => idf(list.len(), num_doc) * (K1 + 1.0),
+        // BM25's doc weight is bounded by K1 + 1 for any freq and doc length,
+        // so query_weight * (K1 + 1) is a valid global bound even when index
+        // stats drift after appends. Keeping it finite matters: an INFINITY
+        // bound can never park the iterator in the WAND tail, forcing a deep
+        // advance on every candidate.
+        let approximate_upper_bound = match &list {
+            PostingList::Compressed(posting) if posting.impacts.is_some() => f32::INFINITY,
+            PostingList::Compressed(posting) if posting.block_size == MAX_POSTING_BLOCK_SIZE => {
+                conservative_bm25_upper_bound(query_weight)
+            }
+            _ => match list.max_score() {
+                Some(max_score) => max_score,
+                None => idf(list.len(), num_doc) * (K1 + 1.0),
+            },
+        };
+        let compressed = match &list {
+            PostingList::Compressed(list) => {
+                Some(UnsafeCell::new(CompressedState::new(list.block_size)))
+            }
+            PostingList::Plain(_) => None,
         };
 
-        let is_compressed = matches!(list, PostingList::Compressed(_));
-
-        Self {
+        let mut posting = Self {
             token,
             token_id,
             position,
@@ -318,9 +839,25 @@ impl PostingIterator {
             list,
             index: 0,
             block_idx: 0,
+            current_doc: None,
             approximate_upper_bound,
-            compressed: is_compressed.then(|| UnsafeCell::new(CompressedState::new())),
-        }
+            use_scorer_upper_bound: false,
+            grouped_terms: None,
+            position_scratch: RefCell::new(Some(Vec::new())),
+            compressed,
+        };
+        posting.refresh_current_doc();
+        posting
+    }
+
+    pub(super) fn with_scorer_upper_bound(mut self) -> Self {
+        self.use_scorer_upper_bound = true;
+        self
+    }
+
+    pub(super) fn with_grouped_terms(mut self, terms: Arc<[GroupedTermScorer]>) -> Self {
+        self.grouped_terms = Some(terms);
+        self
     }
 
     #[inline]
@@ -338,11 +875,53 @@ impl PostingIterator {
         self.approximate_upper_bound
     }
 
+    /// Tightest known list-wide score bound. Impact lists answer from the
+    /// baked doc-weight slab (the data-driven equivalent of the max_score the
+    /// non-impact format bakes at build time); everything else falls back to
+    /// `approximate_upper_bound`. A finite, tight global bound lets lagging
+    /// iterators park in the WAND tail instead of being force-advanced.
     #[inline]
-    fn score<S: Scorer>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
+    fn global_upper_bound<S: Scorer + ?Sized>(&self, scorer: &S) -> f32 {
+        if self.query_weight <= 0.0 {
+            return 0.0;
+        }
+        if let PostingList::Compressed(ref list) = self.list
+            && let Some(impacts) = list.impacts.as_ref()
+        {
+            let compressed = unsafe { &mut *self.compressed_state_ptr() };
+            return self.query_weight
+                * impacts.global_max_doc_weight_cached(
+                    scorer,
+                    &mut compressed.block_max_window.impact_score_cache,
+                );
+        }
+        if self.use_scorer_upper_bound {
+            return scorer_upper_bound(self.query_weight, scorer);
+        }
+        if let PostingList::Compressed(ref list) = self.list
+            && list.block_size == MAX_POSTING_BLOCK_SIZE
+        {
+            return scorer_upper_bound(self.query_weight, scorer);
+        }
+        self.approximate_upper_bound
+    }
+
+    #[inline]
+    fn score<S: Scorer + ?Sized>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
+        if let (Some(grouped_terms), Some(doc)) = (&self.grouped_terms, self.current_doc) {
+            return grouped_terms
+                .iter()
+                .map(|term| term.score(doc.doc_id(), doc_length, scorer))
+                .sum();
+        }
         self.query_weight * scorer.doc_weight(freq, doc_length)
     }
 
+    #[inline]
+    fn has_grouped_terms(&self) -> bool {
+        self.grouped_terms.is_some()
+    }
+
     #[inline]
     fn cost(&self) -> usize {
         self.list.len()
@@ -355,14 +934,19 @@ impl PostingIterator {
 
     #[inline]
     fn doc(&self) -> Option<DocInfo> {
+        self.current_doc
+    }
+
+    fn refresh_current_doc(&mut self) {
         if self.empty() {
-            return None;
+            self.current_doc = None;
+            return;
         }
 
-        match self.list {
+        let current_doc = match self.list {
             PostingList::Compressed(ref list) => {
-                let block_idx = self.index / BLOCK_SIZE;
-                let block_offset = self.index % BLOCK_SIZE;
+                let block_idx = self.index >> list.block_shift();
+                let block_offset = self.index & list.block_mask();
                 let compressed = unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
 
                 // Read from the decompressed block
@@ -372,15 +956,22 @@ impl PostingIterator {
                 Some(doc)
             }
             PostingList::Plain(ref list) => Some(DocInfo::Located(list.doc(self.index))),
-        }
+        };
+        self.current_doc = current_doc;
     }
 
-    fn position_cursor(&self) -> Option<PositionCursor<'_>> {
+    fn position_cursor(&self) -> Result<PositionCursor<'_>> {
         match self.list {
-            PostingList::Plain(ref list) => list.positions.as_ref().map(|positions| {
+            PostingList::Plain(ref list) => {
+                let positions = list.positions.as_ref().ok_or_else(|| {
+                    Error::index(format!(
+                        "positions are missing for token {:?} (token id {}, query position {})",
+                        self.token, self.token_id, self.position
+                    ))
+                })?;
                 let start = positions.value_offsets()[self.index] as usize;
                 let end = positions.value_offsets()[self.index + 1] as usize;
-                PositionCursor::new(
+                Ok(PositionCursor::new(
                     PositionValues::Owned(
                         positions.values().as_primitive::<Int32Type>().values()[start..end]
                             .iter()
@@ -388,48 +979,129 @@ impl PostingIterator {
                             .collect(),
                     ),
                     self.position as i32,
-                )
-            }),
-            PostingList::Compressed(ref list) => match list.positions.as_ref()? {
+                ))
+            }
+            PostingList::Compressed(ref list) => match list.positions.as_ref().ok_or_else(|| {
+                Error::index(format!(
+                    "positions are missing for token {:?} (token id {}, query position {})",
+                    self.token, self.token_id, self.position
+                ))
+            })? {
                 CompressedPositionStorage::LegacyPerDoc(positions) => {
                     let positions = positions.value(self.index);
                     let positions = decompress_positions(positions.as_binary());
-                    Some(PositionCursor::new(
+                    Ok(PositionCursor::new(
                         PositionValues::Owned(positions),
                         self.position as i32,
                     ))
                 }
                 CompressedPositionStorage::SharedStream(stream) => {
-                    let block_idx = self.index / BLOCK_SIZE;
-                    let block_offset = self.index % BLOCK_SIZE;
+                    let block_idx = self.index >> list.block_shift();
+                    let block_offset = self.index & list.block_mask();
                     let compressed =
                         unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
-                    if compressed.position_block_idx != Some(block_idx) {
-                        decode_position_stream_block(
-                            stream.block(block_idx),
-                            compressed.freqs.as_slice(),
-                            stream.codec(),
-                            &mut compressed.position_values,
-                        )
-                        .expect("shared position stream decoding should succeed");
-                        compressed.position_offsets.clear();
-                        compressed
-                            .position_offsets
-                            .reserve(compressed.freqs.len() + 1);
-                        compressed.position_offsets.push(0);
-                        let mut offset = 0usize;
-                        for &freq in &compressed.freqs {
-                            offset += freq as usize;
-                            compressed.position_offsets.push(offset);
+                    match stream.codec() {
+                        PositionStreamCodec::PackedDelta => {
+                            // Seekable layout: decode only the candidate doc's
+                            // positions. Per-block seek state resets when the
+                            // block cursor moves; the group header index and
+                            // varint tail fill in lazily as candidates touch
+                            // them.
+                            if compressed.position_block_idx != Some(block_idx) {
+                                compressed.position_group_offsets.clear();
+                                compressed.position_group_offsets.push(0);
+                                compressed.position_tail.clear();
+                                compressed.position_unpacked_group_idx = None;
+                                compressed.position_offsets.clear();
+                                compressed
+                                    .position_offsets
+                                    .reserve(compressed.freqs.len() + 1);
+                                compressed.position_offsets.push(0);
+                                let mut offset = 0usize;
+                                for &freq in &compressed.freqs {
+                                    offset += freq as usize;
+                                    compressed.position_offsets.push(offset);
+                                }
+                                compressed.position_total_deltas = offset;
+                                compressed.position_block_idx = Some(block_idx);
+                            }
+                            let delta_start = compressed.position_offsets[block_offset];
+                            let delta_end = compressed.position_offsets[block_offset + 1];
+                            let mut position_values = self
+                                .position_scratch
+                                .borrow_mut()
+                                .take()
+                                .unwrap_or_default();
+                            if let Err(error) = seek_packed_doc_positions(
+                                stream.block(block_idx),
+                                compressed.position_total_deltas,
+                                delta_start..delta_end,
+                                &mut compressed.position_group_offsets,
+                                &mut compressed.position_unpacked_group,
+                                &mut compressed.position_unpacked_group_idx,
+                                &mut compressed.position_tail,
+                                &mut position_values,
+                            ) {
+                                *self.position_scratch.borrow_mut() = Some(position_values);
+                                return Err(Error::index(format!(
+                                    "failed to decode positions for token {:?} (token id {}, query position {}) at posting index {}: {error}",
+                                    self.token, self.token_id, self.position, self.index
+                                )));
+                            }
+                            Ok(PositionCursor::new(
+                                PositionValues::Recycled(RecycledPositionValues::new(
+                                    position_values,
+                                    &self.position_scratch,
+                                )),
+                                self.position as i32,
+                            ))
+                        }
+                        PositionStreamCodec::VarintDocDelta => {
+                            if compressed.position_block_idx != Some(block_idx) {
+                                compressed.position_values.clear();
+                                decode_position_stream_block(
+                                    stream.block(block_idx),
+                                    compressed.freqs.as_slice(),
+                                    stream.codec(),
+                                    &mut compressed.position_values,
+                                )
+                                .map_err(|error| {
+                                    Error::index(format!(
+                                        "failed to decode positions for token {:?} (token id {}, query position {}) in block {block_idx}: {error}",
+                                        self.token, self.token_id, self.position
+                                    ))
+                                })?;
+                                compressed.position_offsets.clear();
+                                compressed
+                                    .position_offsets
+                                    .reserve(compressed.freqs.len() + 1);
+                                compressed.position_offsets.push(0);
+                                let mut offset = 0usize;
+                                for &freq in &compressed.freqs {
+                                    offset += freq as usize;
+                                    compressed.position_offsets.push(offset);
+                                }
+                                compressed.position_block_idx = Some(block_idx);
+                            }
+                            let start = compressed.position_offsets[block_offset];
+                            let end = compressed.position_offsets[block_offset + 1];
+                            let mut position_values = self
+                                .position_scratch
+                                .borrow_mut()
+                                .take()
+                                .unwrap_or_default();
+                            position_values.clear();
+                            position_values
+                                .extend_from_slice(&compressed.position_values[start..end]);
+                            Ok(PositionCursor::new(
+                                PositionValues::Recycled(RecycledPositionValues::new(
+                                    position_values,
+                                    &self.position_scratch,
+                                )),
+                                self.position as i32,
+                            ))
                         }
-                        compressed.position_block_idx = Some(block_idx);
                     }
-                    let start = compressed.position_offsets[block_offset];
-                    let end = compressed.position_offsets[block_offset + 1];
-                    Some(PositionCursor::new(
-                        PositionValues::Borrowed(&compressed.position_values[start..end]),
-                        self.position as i32,
-                    ))
                 }
             },
         }
@@ -441,36 +1113,41 @@ impl PostingIterator {
             PostingList::Compressed(ref list) => {
                 debug_assert!(least_id <= u32::MAX as u64);
                 let least_id = least_id as u32;
-                let mut block_idx = self.index / BLOCK_SIZE;
-                while block_idx + 1 < list.blocks.len()
-                    && list.block_least_doc_id(block_idx + 1) <= least_id
-                {
-                    block_idx += 1;
-                }
-                self.index = self.index.max(block_idx * BLOCK_SIZE);
+                let shift = list.block_shift();
+                let block_idx = self.block_idx_for_doc(list, self.index >> shift, least_id);
+                self.index = self.index.max(block_idx << shift);
                 let length = list.length as usize;
                 while self.index < length {
-                    let block_idx = self.index / BLOCK_SIZE;
-                    let block_offset = self.index % BLOCK_SIZE;
+                    let block_idx = self.index >> shift;
+                    let block_offset = self.index & list.block_mask();
                     let compressed =
                         unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
                     let in_block = &compressed.doc_ids[block_offset..];
                     let offset_in_block = in_block.partition_point(|&doc_id| doc_id < least_id);
                     let new_offset = block_offset + offset_in_block;
                     if new_offset < compressed.doc_ids.len() {
-                        self.index = block_idx * BLOCK_SIZE + new_offset;
-                        break;
+                        self.index = (block_idx << shift) + new_offset;
+                        self.block_idx = block_idx;
+                        self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                            doc_id: compressed.doc_ids[new_offset],
+                            frequency: compressed.freqs[new_offset],
+                        }));
+                        return;
                     }
                     if block_idx + 1 >= list.blocks.len() {
                         self.index = length;
+                        self.block_idx = self.index >> shift;
+                        self.current_doc = None;
                         break;
                     }
-                    self.index = (block_idx + 1) * BLOCK_SIZE;
+                    self.index = (block_idx + 1) << shift;
                 }
-                self.block_idx = self.index / BLOCK_SIZE;
+                self.block_idx = self.index >> shift;
+                self.current_doc = None;
             }
             PostingList::Plain(ref list) => {
                 self.index += list.row_ids[self.index..].partition_point(|&id| id < least_id);
+                self.current_doc = (!self.empty()).then(|| DocInfo::Located(list.doc(self.index)));
             }
         }
     }
@@ -480,11 +1157,7 @@ impl PostingIterator {
             PostingList::Compressed(ref list) => {
                 debug_assert!(least_id <= u32::MAX as u64);
                 let least_id = least_id as u32;
-                while self.block_idx + 1 < list.blocks.len()
-                    && list.block_least_doc_id(self.block_idx + 1) <= least_id
-                {
-                    self.block_idx += 1;
-                }
+                self.block_idx = self.block_idx_for_doc(list, self.block_idx, least_id);
             }
             PostingList::Plain(_) => {
                 // we don't have block max score for legacy index,
@@ -493,30 +1166,138 @@ impl PostingIterator {
         }
     }
 
+    /// Anchored level0 impact bound of the current block: (doc_up_to, max_score),
+    /// memoized until the block cursor moves. Malformed entries degrade to
+    /// (u32::MAX, INFINITY), which keeps pruning safe by making the block look
+    /// unskippable.
+    #[inline]
+    fn impact_level0<S: Scorer + ?Sized>(
+        &self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> (u32, f32) {
+        let compressed = unsafe { &mut *self.compressed_state_ptr() };
+        if let Some((block_idx, doc_up_to, score)) = compressed.level0_cache
+            && block_idx == self.block_idx
+        {
+            return (doc_up_to, score);
+        }
+        let doc_up_to = impacts.level0_doc_up_to(self.block_idx).unwrap_or(u32::MAX);
+        let score = impacts.level0_score_cached(
+            self.block_idx,
+            self.query_weight,
+            scorer,
+            &mut compressed.block_max_window.impact_score_cache,
+        );
+        compressed.level0_cache = Some((self.block_idx, doc_up_to, score));
+        (doc_up_to, score)
+    }
+
+    /// Anchored level1 impact bound of the group holding the current block,
+    /// memoized until the cursor crosses a group boundary.
+    #[inline]
+    fn impact_level1<S: Scorer + ?Sized>(
+        &self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> (u32, f32) {
+        let group_idx = self.block_idx / IMPACT_LEVEL1_BLOCKS;
+        let compressed = unsafe { &mut *self.compressed_state_ptr() };
+        if let Some((cached_group_idx, doc_up_to, score)) = compressed.level1_cache
+            && cached_group_idx == group_idx
+        {
+            return (doc_up_to, score);
+        }
+        let doc_up_to = impacts.level1_doc_up_to(group_idx).unwrap_or(u32::MAX);
+        let score = impacts.level1_score_cached(
+            group_idx,
+            self.query_weight,
+            scorer,
+            &mut compressed.block_max_window.impact_score_cache,
+        );
+        compressed.level1_cache = Some((group_idx, doc_up_to, score));
+        (doc_up_to, score)
+    }
+
     #[inline]
-    fn block_max_score(&self) -> f32 {
+    fn block_max_score<S: Scorer + ?Sized>(&self, scorer: &S) -> f32 {
         match self.list {
-            PostingList::Compressed(ref list) => list.block_max_score(self.block_idx),
+            PostingList::Compressed(ref list) => {
+                if let Some(impacts) = list.impacts.as_ref() {
+                    return self.impact_level0(impacts, scorer).1;
+                }
+                if self.use_scorer_upper_bound {
+                    return scorer_upper_bound(self.query_weight, scorer);
+                }
+                if list.block_size == MAX_POSTING_BLOCK_SIZE {
+                    return scorer_upper_bound(self.query_weight, scorer);
+                }
+                list.block_max_score(self.block_idx)
+            }
+            PostingList::Plain(_) if self.use_scorer_upper_bound => {
+                scorer_upper_bound(self.query_weight, scorer)
+            }
             PostingList::Plain(_) => self.approximate_upper_bound,
         }
     }
 
+    /// Tight max-score bound over `[current block, up_to]`. The common case —
+    /// a window ending inside the current block — answers from the anchored
+    /// level0 memo; wider windows fall back to the sliding block-max deque,
+    /// which scores each block once as it slides forward.
     #[inline]
-    fn block_max_score_up_to_with_stats(&mut self, up_to: u64) -> BlockMaxScore {
+    fn block_max_score_up_to_with_stats<S: Scorer + ?Sized>(
+        &self,
+        up_to: u64,
+        scorer: &S,
+    ) -> BlockMaxScore {
         match self.list {
             PostingList::Compressed(ref list) => {
+                if let Some(impacts) = list.impacts.as_ref() {
+                    let (level0_up_to, level0_score) = self.impact_level0(impacts, scorer);
+                    if up_to <= u64::from(level0_up_to) {
+                        return BlockMaxScore {
+                            score: level0_score,
+                            blocks_scanned: 0,
+                        };
+                    }
+                }
+                if self.use_scorer_upper_bound {
+                    return BlockMaxScore {
+                        score: scorer_upper_bound(self.query_weight, scorer),
+                        blocks_scanned: 0,
+                    };
+                }
                 let compressed = unsafe { &mut *self.compressed_state_ptr() };
-                compressed
-                    .block_max_window
-                    .max_score_up_to(list, self.block_idx, up_to)
+                compressed.block_max_window.max_score_up_to(
+                    list,
+                    self.block_idx,
+                    up_to,
+                    self.query_weight,
+                    scorer,
+                )
             }
             PostingList::Plain(_) => BlockMaxScore {
-                score: self.approximate_upper_bound,
+                score: if self.use_scorer_upper_bound {
+                    scorer_upper_bound(self.query_weight, scorer)
+                } else {
+                    self.approximate_upper_bound
+                },
                 blocks_scanned: 0,
             },
         }
     }
 
+    fn window_max_score<S: Scorer + ?Sized>(&self, up_to: Option<u64>, scorer: &S) -> f32 {
+        if let Some(up_to) = up_to
+            && let PostingList::Compressed(ref list) = self.list
+            && list.impacts.is_some()
+        {
+            return self.block_max_score_up_to_with_stats(up_to, scorer).score;
+        }
+        self.block_max_score(scorer)
+    }
+
     #[inline]
     fn is_compressed(&self) -> bool {
         matches!(self.list, PostingList::Compressed(_))
@@ -539,6 +1320,101 @@ impl PostingIterator {
         }
     }
 
+    /// Bulk-score every posting in `[current doc, up_to]` into the window
+    /// accumulator (slot = doc - window_min) and leave the iterator on the
+    /// first doc beyond `up_to`. This is the Lucene `nextDocsAndScores`
+    /// equivalent: it walks the decompressed block arrays directly, with no
+    /// per-doc heap traffic.
+    // The norm cache arg tips this hot-path fn over the limit; bundling the
+    // scoring inputs isn't worth the churn here.
+    #[allow(clippy::too_many_arguments)]
+    fn collect_window_scores<S: Scorer + ?Sized>(
+        &mut self,
+        window_min: u64,
+        up_to: u64,
+        clause_idx: usize,
+        docs: &DocSet,
+        scorer: &S,
+        norm_k: Option<(&[u8], &[f32; 256])>,
+        acc: &mut WindowAccumulator,
+    ) {
+        if self.doc().is_some_and(|doc| doc.doc_id() < window_min) {
+            self.next(window_min);
+        }
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                let shift = list.block_shift();
+                let mask = list.block_mask();
+                'blocks: while let Some(doc) = self.current_doc {
+                    if doc.doc_id() > up_to {
+                        break;
+                    }
+                    let block_idx = self.index >> shift;
+                    let block_offset = self.index & mask;
+                    let compressed =
+                        unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
+                    for offset in block_offset..compressed.doc_ids.len() {
+                        let doc_id = compressed.doc_ids[offset];
+                        if u64::from(doc_id) > up_to {
+                            self.index = (block_idx << shift) + offset;
+                            self.block_idx = block_idx;
+                            self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id,
+                                frequency: compressed.freqs[offset],
+                            }));
+                            break 'blocks;
+                        }
+                        let freq = compressed.freqs[offset];
+                        // One byte-norm load plus a cached addend replaces
+                        // recomputing the BM25 denominator per doc.
+                        let doc_weight = match norm_k {
+                            Some((norms, cache)) => bm25_doc_weight_with_norm(
+                                freq,
+                                cache[norms[doc_id as usize] as usize],
+                            ),
+                            None => scorer.doc_weight(freq, docs.scoring_num_tokens(doc_id)),
+                        };
+                        let score = self.query_weight * doc_weight;
+                        let slot = (u64::from(doc_id) - window_min) as usize;
+                        acc.add(clause_idx, slot, score, freq);
+                    }
+                    // Block exhausted: step into the next block (or finish).
+                    let next_start = (block_idx + 1) << shift;
+                    if next_start >= list.length as usize {
+                        self.index = list.length as usize;
+                        self.block_idx = self.index >> shift;
+                        self.current_doc = None;
+                        break;
+                    }
+                    self.index = next_start;
+                    self.block_idx = block_idx + 1;
+                    let compressed =
+                        unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx + 1) };
+                    self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                        doc_id: compressed.doc_ids[0],
+                        frequency: compressed.freqs[0],
+                    }));
+                }
+            }
+            PostingList::Plain(_) => {
+                while let Some(doc) = self.doc() {
+                    let doc_id = doc.doc_id();
+                    if doc_id > up_to {
+                        break;
+                    }
+                    let doc_length = match &doc {
+                        DocInfo::Raw(raw) => docs.scoring_num_tokens(raw.doc_id),
+                        DocInfo::Located(located) => docs.num_tokens_by_row_id(located.row_id),
+                    };
+                    let score = self.score(scorer, doc.frequency(), doc_length);
+                    let slot = (doc_id - window_min) as usize;
+                    acc.add(clause_idx, slot, score, doc.frequency());
+                    self.next(doc_id + 1);
+                }
+            }
+        }
+    }
+
     #[inline]
     fn next_block_first_doc(&self) -> Option<u64> {
         match self.list {
@@ -553,6 +1429,73 @@ impl PostingIterator {
     }
 }
 
+/// Inner window span (in doc ids) of the bulk MAXSCORE path. Same as Lucene's
+/// `MaxScoreBulkScorer.INNER_WINDOW_SIZE`.
+const MAXSCORE_INNER_WINDOW: usize = 1 << 12;
+
+/// Lucene's `MathUtil.sumUpperBound` factor, adapted to Lance's `f32` score
+/// accumulation. Prefix bounds are summed in `f64`, then widened enough to
+/// cover any recursive `f32` summation order of the same non-negative values.
+#[inline]
+fn score_sum_upper_bound_factor(num_values: usize) -> f64 {
+    if num_values <= 2 {
+        1.0
+    } else {
+        let relative_error_bound = (num_values - 1) as f64 * f64::from(f32::EPSILON);
+        1.0 + 2.0 * relative_error_bound
+    }
+}
+
+#[inline]
+fn score_sum_cannot_exceed(
+    partial_score: f32,
+    remaining_upper_bound: f64,
+    threshold: f32,
+    upper_bound_factor: f64,
+) -> bool {
+    ((f64::from(partial_score) + remaining_upper_bound) * upper_bound_factor) as f32 <= threshold
+}
+
+/// Per-window score/frequency accumulator for the bulk MAXSCORE path. Slot i
+/// covers doc id `window_min + i`; `freqs` is laid out clause-major so accept
+/// paths can recover (term, freq) pairs of the essential clauses.
+struct WindowAccumulator {
+    scores: Vec<f32>,
+    freqs: Vec<u32>,
+    words: Vec<u64>,
+    num_clauses: usize,
+}
+
+impl WindowAccumulator {
+    fn new(num_clauses: usize) -> Self {
+        Self {
+            scores: vec![0.0; MAXSCORE_INNER_WINDOW],
+            freqs: vec![0; num_clauses * MAXSCORE_INNER_WINDOW],
+            words: vec![0; MAXSCORE_INNER_WINDOW / 64],
+            num_clauses,
+        }
+    }
+
+    #[inline]
+    fn add(&mut self, clause_idx: usize, slot: usize, score: f32, freq: u32) {
+        self.scores[slot] += score;
+        // Doc-major layout: one slot's clause frequencies share a cache line.
+        self.freqs[slot * self.num_clauses + clause_idx] = freq;
+        self.words[slot >> 6] |= 1u64 << (slot & 63);
+    }
+
+    #[inline]
+    fn clause_freq(&self, clause_idx: usize, slot: usize) -> u32 {
+        self.freqs[slot * self.num_clauses + clause_idx]
+    }
+
+    #[inline]
+    fn clear_slot(&mut self, slot: usize) {
+        self.scores[slot] = 0.0;
+        self.freqs[slot * self.num_clauses..(slot + 1) * self.num_clauses].fill(0);
+    }
+}
+
 /// How wand identified a candidate: either it already had the real
 /// row_id (DocSet carried row_ids), or only the partition-local
 /// doc_id (deferred-row_id path; the caller must resolve via
@@ -725,6 +1668,11 @@ pub struct Wand<'a, S: Scorer> {
     and_last_doc: Option<u64>,
     and_window_stats: AndWindowStats,
     and_candidates_pruned_before_return: usize,
+    // Test-only override for comparing bulk and classic conjunctions without
+    // mutating the process-wide environment.
+    bulk_and_mode_override: Option<BulkAndMode>,
+    #[cfg(test)]
+    bulk_and_searches: usize,
     docs: &'a DocSet,
     scorer: S,
     // Shared cross-partition top-k floor. Each partition publishes its local
@@ -789,18 +1737,45 @@ impl<'a, S: Scorer> Wand<'a, S> {
             and_last_doc: None,
             and_window_stats: AndWindowStats::default(),
             and_candidates_pruned_before_return: 0,
+            bulk_and_mode_override: None,
+            #[cfg(test)]
+            bulk_and_searches: 0,
             docs,
             scorer,
             shared_threshold: None,
         }
     }
 
+    /// Test hook: force one conjunction mode so parity tests can compare bulk
+    /// and classic search within one process.
+    #[cfg(test)]
+    fn with_bulk_and_mode(mut self, mode: BulkAndMode) -> Self {
+        self.bulk_and_mode_override = Some(mode);
+        self
+    }
+
     /// Share one cross-partition top-k floor across a query's partitions.
     pub(crate) fn with_shared_threshold(mut self, shared: Arc<AtomicU32>) -> Self {
         self.shared_threshold = Some(shared);
         self
     }
 
+    /// Per-search norm→BM25-denominator cache (Lucene's norm cache): the doc
+    /// byte-norm slab plus the 256 possible denominator addends. Available
+    /// when the DocSet scores quantized (256-document-block partitions) and the scorer
+    /// factors `doc_weight` as `(K1+1)*freq/(freq + addend)`. Scoring through
+    /// the cache is bit-identical to `scorer.doc_weight`, because both
+    /// evaluate the same expressions on the same quantized lengths.
+    fn norm_k_cache(&self) -> Option<(&'a [u8], Box<[f32; 256]>)> {
+        let docs: &'a DocSet = self.docs;
+        let norms = docs.scoring_norms()?;
+        let mut cache = Box::new([0f32; 256]);
+        for (code, slot) in cache.iter_mut().enumerate() {
+            *slot = self.scorer.doc_norm(dequantize_doc_length(code as u8))?;
+        }
+        Some((norms, cache))
+    }
+
     /// Set the pruning threshold from this partition's k-th best, raised to the
     /// shared cross-partition floor when one is attached.
     fn update_threshold(&mut self, local_kth: f32, wand_factor: f32) {
@@ -839,15 +1814,48 @@ impl<'a, S: Scorer> Wand<'a, S> {
             return Ok(vec![]);
         }
 
-        match (mask.max_len(), mask.iter_addrs()) {
-            (Some(num_rows_matched), Some(row_ids))
-                if self.operator == Operator::Or
-                    && num_rows_matched * 100
-                        <= FLAT_SEARCH_PERCENT_THRESHOLD.deref() * self.docs.len() as u64 =>
+        if should_flat_search(self.operator, &mask, self.docs.len() as u64) {
+            let row_ids = mask
+                .iter_addrs()
+                .expect("should_flat_search guarantees an iterable mask");
+            return self.flat_search(params, row_ids, metrics);
+        }
+
+        // Top-k disjunctions over compressed lists can opt into the bulk
+        // MAXSCORE path (Lucene MaxScoreBulkScorer style): it streams whole
+        // blocks of the essential clauses into a window accumulator instead of
+        // advancing doc-at-a-time through a heap.
+        if *USE_MAXSCORE_SEARCH
+            && self.operator == Operator::Or
+            && params.phrase_slop.is_none()
+            && !self.head.is_empty()
+            && self.head.iter().all(|posting| {
+                posting.posting.is_compressed() && !posting.posting.has_grouped_terms()
+            })
+        {
+            return self.maxscore_search(params, mask, metrics);
+        }
+
+        // Top-k conjunctions (AND and phrase) over compressed lists use the
+        // bulk path: the same block-max window pruning, but candidates come
+        // from a slice-level merge over decompressed blocks instead of per-doc
+        // `next()` leapfrogging through boxed iterators.
+        if self.operator == Operator::And
+            && !self.lead.is_empty()
+            && self
+                .lead
+                .iter()
+                .all(|posting| posting.is_compressed() && !posting.has_grouped_terms())
+            && self
+                .bulk_and_mode_override
+                .unwrap_or_else(|| *BULK_AND_MODE)
+                .enabled_for(self.lead.len())
+        {
+            #[cfg(test)]
             {
-                return self.flat_search(params, row_ids, metrics);
+                self.bulk_and_searches += 1;
             }
-            _ => {}
+            return self.and_bulk_search(params, mask, metrics);
         }
 
         // Deferred-row_id path: when the DocSet was built without
@@ -856,7 +1864,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
         // row_ids post-wand.
         let docs_has_row_ids = self.docs.has_row_ids();
 
-        let mut candidates = BinaryHeap::with_capacity(std::cmp::min(limit, BLOCK_SIZE * 10));
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
         let mut num_comparisons = 0;
         let mut and_search_stats = (self.operator == Operator::And).then_some(AndSearchStats {
             pruned_before_return_start: self.and_candidates_pruned_before_return,
@@ -903,14 +1911,14 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             let doc_length = match &doc {
-                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
 
             let score = if self.operator == Operator::Or {
                 self.advance_all_tail(doc.doc_id(), Some(doc_length), Some(&mut score));
                 if params.phrase_slop.is_some()
-                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)?
                 {
                     self.push_back_leads(doc.doc_id() + 1);
                     continue;
@@ -919,7 +1927,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             } else {
                 self.advance_all_tail(doc.doc_id(), None, None);
                 if params.phrase_slop.is_some()
-                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)?
                 {
                     continue;
                 }
@@ -929,35 +1937,18 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 self.score(doc_length)
             };
 
-            if candidates.len() < limit {
-                let freqs = self.iter_term_freqs().collect();
+            if candidates.insert(
+                ScoredDoc::new(row_id, score),
+                doc_length,
+                posting_doc_id,
+                self.iter_term_freqs(),
+            )? {
                 if let Some(and_stats) = and_search_stats.as_mut() {
                     and_stats.freqs_collected += 1;
                 }
-                candidates.push(Reverse((
-                    ScoredDoc::new(row_id, score),
-                    freqs,
-                    doc_length,
-                    posting_doc_id,
-                )));
-                if candidates.len() == limit {
-                    let kth = candidates.peek().unwrap().0.0.score.0;
+                if let Some(kth) = candidates.kth_score_if_full() {
                     self.update_threshold(kth, params.wand_factor);
                 }
-            } else if score > candidates.peek().unwrap().0.0.score.0 {
-                let freqs = self.iter_term_freqs().collect();
-                if let Some(and_stats) = and_search_stats.as_mut() {
-                    and_stats.freqs_collected += 1;
-                }
-                candidates.pop();
-                candidates.push(Reverse((
-                    ScoredDoc::new(row_id, score),
-                    freqs,
-                    doc_length,
-                    posting_doc_id,
-                )));
-                let kth = candidates.peek().unwrap().0.0.score.0;
-                self.update_threshold(kth, params.wand_factor);
             }
             if self.operator == Operator::Or {
                 self.push_back_leads(doc.doc_id() + 1);
@@ -995,17 +1986,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 CandidateAddr::Pending(row_id_slot as u32)
             }
         };
-        Ok(candidates
-            .into_iter()
-            .map(
-                |Reverse((doc, freqs, doc_length, posting_doc_id))| DocCandidate {
-                    addr: to_addr(doc.row_id),
-                    posting_doc_id,
-                    freqs,
-                    doc_length,
-                },
-            )
-            .collect())
+        candidates.into_candidates(to_addr)
     }
 
     fn flat_search(
@@ -1018,6 +1999,11 @@ impl<'a, S: Scorer> Wand<'a, S> {
         if limit == 0 {
             return Ok(vec![]);
         }
+        debug_assert!(
+            self.docs.is_empty() || self.docs.supports_reverse_lookup(),
+            "flat_search needs reverse lookups (inv, or sorted legacy row_ids); \
+             the caller picked the wrong DocSet shape for this mask"
+        );
 
         // we need to map the row ids to doc ids, and sort them,
         // because WAND PostingIterator can't go back to the previous doc id.
@@ -1045,7 +2031,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             .unwrap_or(false);
 
         let mut num_comparisons = 0;
-        let mut candidates = BinaryHeap::new();
+        let mut candidates = TopKCollector::new(limit, 0);
         for (doc_id, row_id) in doc_ids {
             num_comparisons += 1;
             self.move_head_before_target_to_tail(doc_id);
@@ -1067,73 +2053,558 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 continue;
             }
 
-            // check positions
-            if params.phrase_slop.is_some()
-                && !self.check_positions(params.phrase_slop.unwrap() as i32)
-            {
-                self.advance_lead_to_head(doc_id + 1);
+            // check positions
+            if params.phrase_slop.is_some()
+                && !self.check_positions(params.phrase_slop.unwrap() as i32)?
+            {
+                self.advance_lead_to_head(doc_id + 1);
+                continue;
+            }
+
+            // score the doc
+            let doc_length = match is_compressed {
+                true => self.docs.scoring_num_tokens(doc_id as u32),
+                false => self.docs.num_tokens_by_row_id(row_id),
+            };
+            if self.operator == Operator::Or && !self.refine_or_candidate(doc_id, doc_length) {
+                // `flat_search` evaluates an explicit allow-list of doc ids. Unlike the
+                // regular WAND path, skipping to the next block boundary is unsafe here
+                // because later doc ids from the same block may still be present in the
+                // allow-list and need to be evaluated individually.
+                self.advance_tail_and_lead_to_head(doc_id + 1);
+                continue;
+            }
+
+            self.collect_tail_matches(doc_id);
+            let score = self.score(doc_length);
+
+            if candidates.insert(
+                ScoredDoc::new(row_id, score),
+                doc_length,
+                doc_id,
+                self.iter_term_freqs(),
+            )? && let Some(kth) = candidates.kth_score_if_full()
+            {
+                self.update_threshold(kth, params.wand_factor);
+            }
+
+            self.advance_lead_to_head(doc_id + 1);
+        }
+        metrics.record_comparisons(num_comparisons);
+
+        // flat_search is driven by an explicit row_ids iterator, so
+        // every candidate already has a real row_id.
+        candidates.into_candidates(CandidateAddr::RowId)
+    }
+
+    /// Bulk MAXSCORE top-k disjunction, mirroring Lucene's MaxScoreBulkScorer.
+    ///
+    /// Per outer window (bounded by the essential clauses' block boundaries):
+    /// clauses are partitioned into a non-essential prefix — sorted by window
+    /// max score, as many as fit under the threshold — and the essential rest.
+    /// Essential clauses stream their postings into a window accumulator in
+    /// bulk; only accumulated candidates that could still beat the threshold
+    /// probe the non-essential clauses. No per-doc heap maintenance happens
+    /// anywhere on this path.
+    fn maxscore_search(
+        &mut self,
+        params: &FtsSearchParams,
+        mask: Arc<RowAddrMask>,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<Vec<DocCandidate>> {
+        struct MaxScoreClause {
+            posting: Box<PostingIterator>,
+            bound: f32,
+            prefix_bound: f64,
+        }
+
+        let limit = params.limit.unwrap_or(usize::MAX);
+        let docs_has_row_ids = self.docs.has_row_ids();
+        let mut clauses = std::mem::take(&mut self.head)
+            .into_vec()
+            .into_iter()
+            .map(|head| MaxScoreClause {
+                posting: head.posting,
+                bound: 0.0,
+                prefix_bound: 0.0,
+            })
+            .collect::<Vec<_>>();
+        let total_sum_upper_bound_factor = score_sum_upper_bound_factor(clauses.len());
+
+        let mut acc = WindowAccumulator::new(clauses.len());
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
+        let norm_k = self.norm_k_cache();
+        let norm_k_ref = norm_k
+            .as_ref()
+            .map(|(norms, cache)| (*norms, cache.as_ref()));
+        let mut num_comparisons = 0usize;
+        // Adaptive minimum window size (Lucene): grow windows when they yield
+        // too few candidates to amortize the per-window bound computations.
+        let mut min_window_size = 1u64;
+        let mut num_windows = 0u64;
+        let mut prev_first_essential = 0usize;
+
+        let mut window_min = clauses
+            .iter()
+            .filter_map(|clause| clause.posting.doc().map(|doc| doc.doc_id()))
+            .min()
+            .unwrap_or(TERMINATED_DOC_ID);
+
+        while window_min < TERMINATED_DOC_ID {
+            clauses.retain(|clause| clause.posting.doc().is_some());
+            if clauses.is_empty() {
+                break;
+            }
+            self.raise_to_shared_floor(params.wand_factor);
+
+            // Window boundary from the previous window's essential clauses
+            // only: dense non-essential clauses must not fragment the window.
+            let first_window_lead = prev_first_essential.min(clauses.len() - 1);
+            let mut window_max = TERMINATED_DOC_ID;
+            for clause in &mut clauses {
+                let doc = clause
+                    .posting
+                    .doc()
+                    .map(|doc| doc.doc_id())
+                    .expect("exhausted clauses were retained out");
+                clause.posting.shallow_next(doc.max(window_min));
+            }
+            for clause in &clauses[first_window_lead..] {
+                window_max = window_max.min(clause.posting.block_end_doc());
+            }
+            if clauses.len() > 1 {
+                // Target at least 32 candidates per clause per window on
+                // average before shrinking windows back to block granularity.
+                if (num_comparisons as u64) < num_windows * 32 * clauses.len() as u64 {
+                    min_window_size = (min_window_size * 2).min(MAXSCORE_INNER_WINDOW as u64);
+                } else {
+                    min_window_size = 1;
+                }
+                window_max = window_max.max(window_min.saturating_add(min_window_size - 1));
+            }
+
+            for clause in &mut clauses {
+                let doc = clause
+                    .posting
+                    .doc()
+                    .map(|doc| doc.doc_id())
+                    .expect("exhausted clauses were retained out");
+                clause.bound = if doc > window_max {
+                    0.0
+                } else {
+                    clause
+                        .posting
+                        .block_max_score_up_to_with_stats(window_max, &self.scorer)
+                        .score
+                };
+            }
+            clauses.sort_unstable_by(|a, b| a.bound.total_cmp(&b.bound));
+            let mut first_essential = 0;
+            let mut prefix = 0.0_f64;
+            if self.threshold > 0.0 {
+                for (i, clause) in clauses.iter_mut().enumerate() {
+                    let next_prefix = prefix + f64::from(clause.bound);
+                    let widened = (next_prefix * score_sum_upper_bound_factor(i + 1)) as f32;
+                    if widened > self.threshold {
+                        break;
+                    }
+                    prefix = next_prefix;
+                    clause.prefix_bound = prefix;
+                    first_essential = i + 1;
+                }
+            }
+            prev_first_essential = first_essential;
+            num_windows += 1;
+
+            if first_essential == clauses.len() {
+                // No clause combination inside this window can beat the
+                // threshold: skip it wholesale.
+                window_min = match window_max {
+                    TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                    max => max + 1,
+                };
+                // Single live clause: instead of re-running the window
+                // machinery once per block, scan the baked per-block bounds
+                // for the next block that can beat the threshold. This is the
+                // slab form of Lucene's getSkipUpTo and turns dead stretches
+                // of a dominant term into a tight load-mul-compare loop.
+                if clauses.len() == 1 && window_min != TERMINATED_DOC_ID && self.threshold > 0.0 {
+                    let posting = &clauses[0].posting;
+                    if let PostingList::Compressed(ref list) = posting.list
+                        && let Some(impacts) = list.impacts.as_ref()
+                    {
+                        let compressed = unsafe { &mut *posting.compressed_state_ptr() };
+                        let bounds = impacts.level0_doc_weight_bounds_cached(
+                            &self.scorer,
+                            &mut compressed.block_max_window.impact_score_cache,
+                        );
+                        let query_weight = posting.query_weight;
+                        // Position by binary search on the first-doc slab: the
+                        // deep cursor lags arbitrarily far behind during long
+                        // skip runs, and walking from it re-scans the same
+                        // blocks on every dead window.
+                        let first_docs = list.block_first_docs();
+                        let mut block_idx = first_docs
+                            .partition_point(|&first| u64::from(first) <= window_min)
+                            .saturating_sub(1);
+                        while block_idx < bounds.len()
+                            && query_weight * bounds[block_idx] <= self.threshold
+                        {
+                            block_idx += 1;
+                        }
+                        window_min = if block_idx < bounds.len() {
+                            window_min.max(u64::from(list.block_least_doc_id(block_idx)))
+                        } else {
+                            TERMINATED_DOC_ID
+                        };
+                    }
+                }
+                continue;
+            }
+
+            let total_non_essential_bound = if first_essential > 0 {
+                clauses[first_essential - 1].prefix_bound
+            } else {
+                0.0
+            };
+
+            // Single essential clause (the common case once the threshold is
+            // competitive): stream it directly against the non-essential
+            // prefix, skipping the accumulator entirely.
+            if first_essential + 1 == clauses.len() {
+                let (non_essential, essential) = clauses.split_at_mut(first_essential);
+                let posting = &mut essential[0].posting;
+                if posting.doc().is_some_and(|doc| doc.doc_id() < window_min) {
+                    posting.next(window_min);
+                }
+                let essential_term = posting.term_index();
+                let essential_weight = posting.query_weight;
+
+                macro_rules! consider_candidate {
+                    ($doc:expr, $freq:expr) => {{
+                        let doc = $doc;
+                        let freq = $freq;
+                        num_comparisons += 1;
+                        // One byte-norm load + cached addend when available;
+                        // the exact doc length is only needed at insert time.
+                        let norm_addend =
+                            norm_k_ref.map(|(norms, cache)| cache[norms[doc as usize] as usize]);
+                        let score = match norm_addend {
+                            Some(addend) => {
+                                essential_weight * bm25_doc_weight_with_norm(freq, addend)
+                            }
+                            None => {
+                                essential_weight
+                                    * self
+                                        .scorer
+                                        .doc_weight(freq, self.docs.scoring_num_tokens(doc as u32))
+                            }
+                        };
+                        if !(self.threshold > 0.0
+                            && score_sum_cannot_exceed(
+                                score,
+                                total_non_essential_bound,
+                                self.threshold,
+                                total_sum_upper_bound_factor,
+                            ))
+                        {
+                            let row_id = if docs_has_row_ids {
+                                self.docs.row_id(doc as u32)
+                            } else {
+                                doc
+                            };
+                            let masked_out = docs_has_row_ids
+                                && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id));
+                            if !masked_out {
+                                let mut total = score;
+                                let mut rejected = false;
+                                for i in (0..non_essential.len()).rev() {
+                                    if self.threshold > 0.0
+                                        && score_sum_cannot_exceed(
+                                            total,
+                                            non_essential[i].prefix_bound,
+                                            self.threshold,
+                                            total_sum_upper_bound_factor,
+                                        )
+                                    {
+                                        rejected = true;
+                                        break;
+                                    }
+                                    let probe = &mut non_essential[i].posting;
+                                    if probe.doc().is_some_and(|d| d.doc_id() < doc) {
+                                        probe.next(doc);
+                                    }
+                                    if let Some(d) = probe.doc()
+                                        && d.doc_id() == doc
+                                    {
+                                        total += match norm_addend {
+                                            Some(addend) => {
+                                                probe.query_weight
+                                                    * bm25_doc_weight_with_norm(
+                                                        d.frequency(),
+                                                        addend,
+                                                    )
+                                            }
+                                            None => probe.score(
+                                                &self.scorer,
+                                                d.frequency(),
+                                                self.docs.scoring_num_tokens(doc as u32),
+                                            ),
+                                        };
+                                    }
+                                }
+
+                                // Match the classic path's emission rule: a
+                                // candidate must beat the running threshold,
+                                // which drops zero-score matches (e.g. terms
+                                // with idf 0) exactly like Wand::next does.
+                                if !rejected && total > self.threshold {
+                                    let doc_length = self.docs.scoring_num_tokens(doc as u32);
+                                    if candidates.insert(
+                                        ScoredDoc::new(row_id, total),
+                                        doc_length,
+                                        doc,
+                                        std::iter::once((essential_term, freq)).chain(
+                                            non_essential.iter().filter_map(|clause| {
+                                                clause.posting.doc().and_then(|d| {
+                                                    (d.doc_id() == doc).then(|| {
+                                                        (clause.posting.term_index(), d.frequency())
+                                                    })
+                                                })
+                                            }),
+                                        ),
+                                    )? && let Some(kth) = candidates.kth_score_if_full()
+                                    {
+                                        self.update_threshold(kth, params.wand_factor);
+                                    }
+                                }
+                            }
+                        }
+                    }};
+                }
+
+                match posting.list {
+                    PostingList::Compressed(ref list) => {
+                        let shift = list.block_shift();
+                        let mask = list.block_mask();
+                        'stream: while let Some(cur) = posting.current_doc {
+                            if cur.doc_id() > window_max {
+                                break;
+                            }
+                            let block_idx = posting.index >> shift;
+                            let block_offset = posting.index & mask;
+                            let compressed = unsafe {
+                                &mut *posting.ensure_compressed_block_ptr(list, block_idx)
+                            };
+                            for offset in block_offset..compressed.doc_ids.len() {
+                                let doc_id = compressed.doc_ids[offset];
+                                if u64::from(doc_id) > window_max {
+                                    posting.index = (block_idx << shift) + offset;
+                                    posting.block_idx = block_idx;
+                                    posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                        doc_id,
+                                        frequency: compressed.freqs[offset],
+                                    }));
+                                    break 'stream;
+                                }
+                                consider_candidate!(u64::from(doc_id), compressed.freqs[offset]);
+                            }
+                            let next_start = (block_idx + 1) << shift;
+                            if next_start >= list.length as usize {
+                                posting.index = list.length as usize;
+                                posting.block_idx = posting.index >> shift;
+                                posting.current_doc = None;
+                                break;
+                            }
+                            posting.index = next_start;
+                            posting.block_idx = block_idx + 1;
+                            let compressed = unsafe {
+                                &mut *posting.ensure_compressed_block_ptr(list, block_idx + 1)
+                            };
+                            posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id: compressed.doc_ids[0],
+                                frequency: compressed.freqs[0],
+                            }));
+                        }
+                    }
+                    PostingList::Plain(_) => {
+                        while let Some(cur) = posting.doc() {
+                            let doc = cur.doc_id();
+                            if doc > window_max {
+                                break;
+                            }
+                            consider_candidate!(doc, cur.frequency());
+                            posting.next(doc + 1);
+                        }
+                    }
+                }
+
+                window_min = match window_max {
+                    TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                    max => max + 1,
+                };
                 continue;
             }
 
-            // score the doc
-            let doc_length = match is_compressed {
-                true => self.docs.num_tokens(doc_id as u32),
-                false => self.docs.num_tokens_by_row_id(row_id),
-            };
-            if self.operator == Operator::Or && !self.refine_or_candidate(doc_id, doc_length) {
-                // `flat_search` evaluates an explicit allow-list of doc ids. Unlike the
-                // regular WAND path, skipping to the next block boundary is unsafe here
-                // because later doc ids from the same block may still be present in the
-                // allow-list and need to be evaluated individually.
-                self.advance_tail_and_lead_to_head(doc_id + 1);
-                continue;
-            }
+            // Stream the essential clauses through inner windows.
+            let mut inner_min = window_min;
+            loop {
+                let mut next_essential_doc = TERMINATED_DOC_ID;
+                for clause in &clauses[first_essential..] {
+                    if let Some(doc) = clause.posting.doc() {
+                        next_essential_doc = next_essential_doc.min(doc.doc_id());
+                    }
+                }
+                inner_min = inner_min.max(next_essential_doc);
+                if inner_min == TERMINATED_DOC_ID || inner_min > window_max {
+                    break;
+                }
+                let inner_max =
+                    window_max.min(inner_min.saturating_add(MAXSCORE_INNER_WINDOW as u64 - 1));
+
+                for (clause_idx, clause) in clauses.iter_mut().enumerate().skip(first_essential) {
+                    clause.posting.collect_window_scores(
+                        inner_min,
+                        inner_max,
+                        clause_idx,
+                        self.docs,
+                        &self.scorer,
+                        norm_k_ref,
+                        &mut acc,
+                    );
+                }
 
-            self.collect_tail_matches(doc_id);
-            let score = self.score(doc_length);
+                // Drain candidates in doc order, completing them with the
+                // non-essential clauses ordered by descending bound.
+                for word_idx in 0..acc.words.len() {
+                    let mut word = acc.words[word_idx];
+                    if word == 0 {
+                        continue;
+                    }
+                    acc.words[word_idx] = 0;
+                    while word != 0 {
+                        let bit = word.trailing_zeros() as usize;
+                        word &= word - 1;
+                        let slot = (word_idx << 6) | bit;
+                        let doc = inner_min + slot as u64;
+                        let mut score = acc.scores[slot];
+                        num_comparisons += 1;
+
+                        if self.threshold > 0.0
+                            && score_sum_cannot_exceed(
+                                score,
+                                total_non_essential_bound,
+                                self.threshold,
+                                total_sum_upper_bound_factor,
+                            )
+                        {
+                            acc.clear_slot(slot);
+                            continue;
+                        }
 
-            if candidates.len() < limit {
-                let freqs = self.iter_term_freqs().collect();
-                candidates.push(Reverse((
-                    ScoredDoc::new(row_id, score),
-                    freqs,
-                    doc_length,
-                    doc_id,
-                )));
-                if candidates.len() == limit {
-                    let kth = candidates.peek().unwrap().0.0.score.0;
-                    self.update_threshold(kth, params.wand_factor);
+                        let row_id = if docs_has_row_ids {
+                            self.docs.row_id(doc as u32)
+                        } else {
+                            doc
+                        };
+                        if docs_has_row_ids
+                            && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id))
+                        {
+                            acc.clear_slot(slot);
+                            continue;
+                        }
+
+                        // Doc length is only needed at heap-insert time; the
+                        // non-essential completion scores go through the norm
+                        // cache when available.
+                        let norm_addend =
+                            norm_k_ref.map(|(norms, cache)| cache[norms[doc as usize] as usize]);
+                        let mut doc_length_cell: Option<u32> = None;
+                        let mut rejected = false;
+                        for i in (0..first_essential).rev() {
+                            if self.threshold > 0.0
+                                && score_sum_cannot_exceed(
+                                    score,
+                                    clauses[i].prefix_bound,
+                                    self.threshold,
+                                    total_sum_upper_bound_factor,
+                                )
+                            {
+                                rejected = true;
+                                break;
+                            }
+                            let posting = &mut clauses[i].posting;
+                            if posting.doc().is_some_and(|d| d.doc_id() < doc) {
+                                posting.next(doc);
+                            }
+                            if let Some(d) = posting.doc()
+                                && d.doc_id() == doc
+                            {
+                                score += match norm_addend {
+                                    Some(addend) => {
+                                        posting.query_weight
+                                            * bm25_doc_weight_with_norm(d.frequency(), addend)
+                                    }
+                                    None => {
+                                        let doc_length =
+                                            *doc_length_cell.get_or_insert_with(|| {
+                                                self.docs.scoring_num_tokens(doc as u32)
+                                            });
+                                        posting.score(&self.scorer, d.frequency(), doc_length)
+                                    }
+                                };
+                            }
+                        }
+
+                        if !rejected && score > self.threshold {
+                            let doc_length = doc_length_cell
+                                .unwrap_or_else(|| self.docs.scoring_num_tokens(doc as u32));
+                            if candidates.insert(
+                                ScoredDoc::new(row_id, score),
+                                doc_length,
+                                doc,
+                                clauses.iter().enumerate().filter_map(|(i, clause)| {
+                                    if i >= first_essential {
+                                        let freq = acc.clause_freq(i, slot);
+                                        (freq > 0).then(|| (clause.posting.term_index(), freq))
+                                    } else {
+                                        clause.posting.doc().and_then(|d| {
+                                            (d.doc_id() == doc).then(|| {
+                                                (clause.posting.term_index(), d.frequency())
+                                            })
+                                        })
+                                    }
+                                }),
+                            )? && let Some(kth) = candidates.kth_score_if_full()
+                            {
+                                self.update_threshold(kth, params.wand_factor);
+                            }
+                        }
+                        acc.clear_slot(slot);
+                    }
                 }
-            } else if score > candidates.peek().unwrap().0.0.score.0 {
-                let freqs = self.iter_term_freqs().collect();
-                candidates.pop();
-                candidates.push(Reverse((
-                    ScoredDoc::new(row_id, score),
-                    freqs,
-                    doc_length,
-                    doc_id,
-                )));
-                let kth = candidates.peek().unwrap().0.0.score.0;
-                self.update_threshold(kth, params.wand_factor);
+                if inner_max >= window_max {
+                    break;
+                }
+                inner_min = inner_max + 1;
             }
 
-            self.advance_lead_to_head(doc_id + 1);
+            window_min = match window_max {
+                TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                max => max + 1,
+            };
         }
+
         metrics.record_comparisons(num_comparisons);
 
-        // flat_search is driven by an explicit row_ids iterator, so
-        // every candidate already has a real row_id.
-        Ok(candidates
-            .into_iter()
-            .map(
-                |Reverse((doc, freqs, doc_length, posting_doc_id))| DocCandidate {
-                    addr: CandidateAddr::RowId(doc.row_id),
-                    posting_doc_id,
-                    freqs,
-                    doc_length,
-                },
-            )
-            .collect())
+        let to_addr = |row_id_slot: u64| {
+            if docs_has_row_ids {
+                CandidateAddr::RowId(row_id_slot)
+            } else {
+                CandidateAddr::Pending(row_id_slot as u32)
+            }
+        };
+        candidates.into_candidates(to_addr)
     }
 
     // calculate the score of the current document
@@ -1174,7 +2645,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
         let remaining_upper_bound = remaining
             .iter()
-            .map(|posting| posting.block_max_score())
+            .map(|posting| posting.block_max_score(&self.scorer))
             .sum::<f32>();
         first.score(&self.scorer, doc.frequency(), doc_length) + remaining_upper_bound
             <= self.threshold
@@ -1214,10 +2685,16 @@ impl<'a, S: Scorer> Wand<'a, S> {
             if self.threshold > 0.0 && self.or_block_window_max() <= self.threshold {
                 // On the final block `up_to` is the `u64::MAX` sentinel; step once
                 // there to avoid seeking past the valid doc id range.
-                let skip_to = match self.up_to {
+                let mut skip_to = match self.up_to {
                     Some(up_to) if up_to < u32::MAX as u64 => up_to + 1,
                     _ => target + 1,
                 };
+                // The narrow window is dead; if the whole level1 group is dead
+                // too, hop over it in one advance.
+                let group_skip = self.or_group_skip_to();
+                if let Some(group_skip_to) = group_skip {
+                    skip_to = skip_to.max(group_skip_to);
+                }
                 self.push_back_leads(skip_to);
                 continue;
             }
@@ -1227,7 +2704,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 continue;
             };
             let doc_length = match &first_doc {
-                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
             let mut lead_score = 0.0;
@@ -1309,7 +2786,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
             let lead_doc = self.lead.first().and_then(|posting| posting.doc())?;
             let doc_length = match &lead_doc {
-                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
             if self.and_candidate_cannot_beat_threshold(doc_length) {
@@ -1335,6 +2812,577 @@ impl<'a, S: Scorer> Wand<'a, S> {
             .max(target)
     }
 
+    /// Bulk conjunction search. The window ends at the nearest next-block
+    /// boundary across the clauses, so within a window every clause
+    /// contributes exactly one decompressed block and the intersection is a
+    /// plain merge over `u32` slices — the per-candidate cost drops from a
+    /// full `PostingIterator::next` call per clause to a couple of loads.
+    /// Window skipping, per-candidate pruning, scoring, and heap semantics
+    /// mirror the classic loop exactly, so results are identical.
+    fn and_bulk_search(
+        &mut self,
+        params: &FtsSearchParams,
+        mask: Arc<RowAddrMask>,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<Vec<DocCandidate>> {
+        let limit = params.limit.unwrap_or(usize::MAX);
+        if limit == 0 {
+            return Ok(vec![]);
+        }
+        let docs_has_row_ids = self.docs.has_row_ids();
+        let num_lists = self.lead.len();
+        let phrase_slop = params.phrase_slop;
+
+        // Per-window view of one clause's current block. Raw pointers into the
+        // clause's `CompressedState`; valid for the whole window because the
+        // block cursor does not move within a window (position decoding writes
+        // to separate fields of the same state).
+        struct WindowList {
+            docs: *const u32,
+            freqs: *const u32,
+            // Cursor and exclusive end, as offsets within the block.
+            pos: usize,
+            end: usize,
+            // Absolute posting index of the block's first entry.
+            block_start: usize,
+        }
+
+        // Merge kernels: intersect the window's per-clause slices and record
+        // each match as (doc, per-clause block offsets). Hand-specialized for
+        // two and three clauses so cursors and bounds stay in registers; the
+        // generic kernel covers other widths. Offsets fit u8 because a block
+        // holds at most `MAX_POSTING_BLOCK_SIZE` (256) entries. The macro
+        // stamps a scalar and an AVX2 variant — `#[target_feature]` must
+        // cover the whole kernel for the vector catch-up scans to inline.
+        // Kernels prune a lead doc before any follower advance when its
+        // frequency-bucketed score bound (plus the other clauses' block
+        // maxes) cannot beat the threshold — the score-first ordering
+        // Lucene's conjunction scorer uses, with doc length dropped from the
+        // bound so no doc-length load is involved. The bound is monotone
+        // (doc length only shrinks a BM25 weight, and the last bucket holds
+        // the clause sup), so every skipped doc would also fail the exact
+        // per-candidate prune: results are unchanged, the work never happens.
+        macro_rules! merge_kernels {
+            ($name2:ident, $name3:ident, $geq:ident $(, #[$feat:meta])?) => {
+                $(#[$feat])?
+                unsafe fn $name2(
+                    wins: &[WindowList],
+                    lut: &[f32; FREQ_LUT_BUCKETS],
+                    others_block_max: f32,
+                    threshold: f32,
+                    docs_out: &mut Vec<u32>,
+                    offs_out: &mut Vec<u8>,
+                ) {
+                    let (d0, mut p0, e0) = (wins[0].docs, wins[0].pos, wins[0].end);
+                    let (d1, mut p1, e1) = (wins[1].docs, wins[1].pos, wins[1].end);
+                    let f0 = wins[0].freqs;
+                    let prune = threshold > f32::NEG_INFINITY;
+                    unsafe {
+                        while p0 < e0 {
+                            let doc = *d0.add(p0);
+                            if prune {
+                                let freq = (*f0.add(p0) as usize).min(FREQ_LUT_BUCKETS - 1);
+                                if lut[freq] + others_block_max <= threshold {
+                                    p0 += 1;
+                                    continue;
+                                }
+                            }
+                            p1 = $geq(d1, p1, e1, doc);
+                            if p1 >= e1 {
+                                return;
+                            }
+                            let second = *d1.add(p1);
+                            if second > doc {
+                                p0 = $geq(d0, p0 + 1, e0, second);
+                                continue;
+                            }
+                            docs_out.push(doc);
+                            offs_out.push(p0 as u8);
+                            offs_out.push(p1 as u8);
+                            p0 += 1;
+                        }
+                    }
+                }
+
+                $(#[$feat])?
+                unsafe fn $name3(
+                    wins: &[WindowList],
+                    lut: &[f32; FREQ_LUT_BUCKETS],
+                    others_block_max: f32,
+                    threshold: f32,
+                    docs_out: &mut Vec<u32>,
+                    offs_out: &mut Vec<u8>,
+                ) {
+                    let (d0, mut p0, e0) = (wins[0].docs, wins[0].pos, wins[0].end);
+                    let (d1, mut p1, e1) = (wins[1].docs, wins[1].pos, wins[1].end);
+                    let (d2, mut p2, e2) = (wins[2].docs, wins[2].pos, wins[2].end);
+                    let f0 = wins[0].freqs;
+                    let prune = threshold > f32::NEG_INFINITY;
+                    unsafe {
+                        'outer: while p0 < e0 {
+                            let doc = *d0.add(p0);
+                            if prune {
+                                let freq = (*f0.add(p0) as usize).min(FREQ_LUT_BUCKETS - 1);
+                                if lut[freq] + others_block_max <= threshold {
+                                    p0 += 1;
+                                    continue 'outer;
+                                }
+                            }
+                            p1 = $geq(d1, p1, e1, doc);
+                            if p1 >= e1 {
+                                return;
+                            }
+                            let second = *d1.add(p1);
+                            if second > doc {
+                                p0 = $geq(d0, p0 + 1, e0, second);
+                                continue 'outer;
+                            }
+                            p2 = $geq(d2, p2, e2, doc);
+                            if p2 >= e2 {
+                                return;
+                            }
+                            let third = *d2.add(p2);
+                            if third > doc {
+                                p0 = $geq(d0, p0 + 1, e0, third);
+                                continue 'outer;
+                            }
+                            docs_out.push(doc);
+                            offs_out.push(p0 as u8);
+                            offs_out.push(p1 as u8);
+                            offs_out.push(p2 as u8);
+                            p0 += 1;
+                        }
+                    }
+                }
+            };
+        }
+        merge_kernels!(merge_window_2, merge_window_3, find_next_geq_scalar);
+        #[cfg(target_arch = "x86_64")]
+        merge_kernels!(
+            merge_window_2_avx2,
+            merge_window_3_avx2,
+            find_next_geq_avx2,
+            #[target_feature(enable = "avx2")]
+        );
+
+        #[inline]
+        fn merge_window_1(wins: &[WindowList], docs_out: &mut Vec<u32>, offs_out: &mut Vec<u8>) {
+            let win = &wins[0];
+            for pos in win.pos..win.end {
+                docs_out.push(unsafe { *win.docs.add(pos) });
+                offs_out.push(pos as u8);
+            }
+        }
+
+        #[inline]
+        #[allow(clippy::too_many_arguments)]
+        fn merge_window_n(
+            wins: &[WindowList],
+            lut: &[f32; FREQ_LUT_BUCKETS],
+            others_block_max: f32,
+            threshold: f32,
+            cursors: &mut Vec<usize>,
+            docs_out: &mut Vec<u32>,
+            offs_out: &mut Vec<u8>,
+        ) {
+            let prune = threshold > f32::NEG_INFINITY;
+            cursors.clear();
+            cursors.extend(wins.iter().map(|win| win.pos));
+            'outer: while cursors[0] < wins[0].end {
+                let doc = unsafe { *wins[0].docs.add(cursors[0]) };
+                if prune {
+                    let freq = unsafe { *wins[0].freqs.add(cursors[0]) as usize }
+                        .min(FREQ_LUT_BUCKETS - 1);
+                    if lut[freq] + others_block_max <= threshold {
+                        cursors[0] += 1;
+                        continue 'outer;
+                    }
+                }
+                for j in 1..wins.len() {
+                    let win = &wins[j];
+                    let pos = unsafe { find_next_geq(win.docs, cursors[j], win.end, doc) };
+                    cursors[j] = pos;
+                    if pos >= win.end {
+                        return;
+                    }
+                    let clause_doc = unsafe { *win.docs.add(pos) };
+                    if clause_doc > doc {
+                        cursors[0] = unsafe {
+                            find_next_geq(wins[0].docs, cursors[0] + 1, wins[0].end, clause_doc)
+                        };
+                        continue 'outer;
+                    }
+                }
+                docs_out.push(doc);
+                for &pos in cursors.iter() {
+                    offs_out.push(pos as u8);
+                }
+                cursors[0] += 1;
+            }
+        }
+
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
+        let mut num_comparisons: usize = 0;
+        let mut stats = AndSearchStats {
+            pruned_before_return_start: self.and_candidates_pruned_before_return,
+            ..Default::default()
+        };
+        let mut wins: Vec<WindowList> = Vec::with_capacity(num_lists);
+        // Per-window candidate batch. The merge kernel only records matches;
+        // scoring then runs in two passes so the doc-length gather issues
+        // independent loads (their cache misses overlap) instead of
+        // serializing behind per-candidate branching. A window spans at most
+        // one block per clause, so a batch holds at most one block's worth.
+        let mut batch_docs: Vec<u32> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut batch_offs: Vec<u8> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE * num_lists);
+        let mut batch_lens: Vec<u32> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut batch_norms: Vec<u8> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut cursor_scratch: Vec<usize> = Vec::with_capacity(num_lists);
+        // Norm cache: one byte-norm load plus a cached addend replaces the
+        // per-clause BM25 denominator recompute in pass B.
+        let norm_k = self.norm_k_cache();
+        let norm_k_ref = norm_k
+            .as_ref()
+            .map(|(norms, cache)| (*norms, cache.as_ref()));
+
+        // Per-window prune LUT for the merge kernels: an upper bound of the
+        // first (rarest) clause's score by clamped frequency. Lead docs whose
+        // bound plus the remaining clauses' block maxes cannot beat the
+        // threshold are skipped before any follower advances — the same
+        // score-first ordering Lucene's conjunction scorer uses, with the
+        // doc-length dropped from the bound so no doc-length load is needed.
+        // The last bucket holds the frequency-independent sup, so clamping
+        // stays a valid upper bound. Skips only avoid work; every emitted
+        // candidate still goes through the exact per-candidate prune below.
+        const FREQ_LUT_BUCKETS: usize = 64;
+        let mut freq_bound_lut = [f32::INFINITY; FREQ_LUT_BUCKETS];
+        for (freq, slot) in freq_bound_lut
+            .iter_mut()
+            .enumerate()
+            .take(FREQ_LUT_BUCKETS - 1)
+        {
+            *slot = self.lead[0].score(&self.scorer, freq as u32, 0);
+        }
+        // The clamp bucket must bound every frequency it absorbs; the
+        // clause-wide sup does.
+        freq_bound_lut[FREQ_LUT_BUCKETS - 1] = self.lead[0].approximate_upper_bound();
+
+        // The conjunction can only start at the max of the clauses' first docs.
+        let mut target: u64 = 0;
+        for posting in &self.lead {
+            match posting.doc() {
+                Some(doc) => target = target.max(doc.doc_id()),
+                None => return Ok(vec![]),
+            }
+        }
+
+        'window: loop {
+            self.raise_to_shared_floor(params.wand_factor);
+            if self.threshold > 0.0 {
+                let advanced = self.and_advance_target(target);
+                if advanced == TERMINATED_DOC_ID {
+                    break;
+                }
+                target = advanced;
+            }
+            debug_assert!(target <= u32::MAX as u64);
+            let target32 = target as u32;
+
+            // Position every clause's block cursor at the block that can hold
+            // `target`, and end the window at the nearest next-block boundary.
+            let mut win_end = TERMINATED_DOC_ID;
+            for j in 0..num_lists {
+                let (block_idx, block_up_to) = {
+                    let posting = &self.lead[j];
+                    let PostingList::Compressed(ref list) = posting.list else {
+                        unreachable!("bulk AND requires compressed postings");
+                    };
+                    let block_idx = posting.block_idx_for_doc(list, posting.block_idx, target32);
+                    let block_up_to = if block_idx + 1 < list.blocks.len() {
+                        u64::from(list.block_least_doc_id(block_idx + 1)).saturating_sub(1)
+                    } else {
+                        TERMINATED_DOC_ID
+                    };
+                    (block_idx, block_up_to.max(target))
+                };
+                self.lead[j].block_idx = block_idx;
+                win_end = win_end.min(block_up_to);
+            }
+            let win_end32 = u32::try_from(win_end).unwrap_or(u32::MAX);
+
+            // Decompress each clause's block and slice it to [target, win_end].
+            wins.clear();
+            let mut skip_window = false;
+            let mut exhausted = false;
+            for posting in &self.lead {
+                let PostingList::Compressed(ref list) = posting.list else {
+                    unreachable!("bulk AND requires compressed postings");
+                };
+                let block_idx = posting.block_idx;
+                let state = unsafe { &mut *posting.ensure_compressed_block_ptr(list, block_idx) };
+                let lo = state.doc_ids.partition_point(|&doc| doc < target32);
+                let hi = if win_end32 == u32::MAX {
+                    state.doc_ids.len()
+                } else {
+                    lo + state.doc_ids[lo..].partition_point(|&doc| doc <= win_end32)
+                };
+                if lo == hi {
+                    // No docs of this clause in the window: the whole window
+                    // has no conjunction match. If this was the clause's last
+                    // block and it is fully behind the target, the clause is
+                    // exhausted and the conjunction is done.
+                    if block_idx + 1 >= list.blocks.len()
+                        && state.doc_ids.last().is_none_or(|&doc| doc < target32)
+                    {
+                        exhausted = true;
+                    }
+                    skip_window = true;
+                    break;
+                }
+                wins.push(WindowList {
+                    docs: state.doc_ids.as_ptr(),
+                    freqs: state.freqs.as_ptr(),
+                    pos: lo,
+                    end: hi,
+                    block_start: block_idx << list.block_shift(),
+                });
+            }
+            if exhausted {
+                break 'window;
+            }
+            if !skip_window {
+                // Constant within the window (block-anchored); mirrors
+                // `and_candidate_cannot_beat_threshold`'s remaining-clause
+                // bound of first-clause-exact + rest-block-max.
+                let others_block_max: f32 = self.lead[1..]
+                    .iter()
+                    .map(|posting| posting.block_max_score(&self.scorer))
+                    .sum();
+
+                batch_docs.clear();
+                batch_offs.clear();
+                // NEG_INFINITY disables the kernel-level freq-bound prune
+                // (single clause, or no threshold yet).
+                let kernel_threshold = if self.threshold > 0.0 && num_lists >= 2 {
+                    self.threshold
+                } else {
+                    f32::NEG_INFINITY
+                };
+                #[cfg(target_arch = "x86_64")]
+                let use_avx2 = *HAS_AVX2;
+                #[cfg(not(target_arch = "x86_64"))]
+                let use_avx2 = false;
+                match (num_lists, use_avx2) {
+                    (1, _) => merge_window_1(&wins, &mut batch_docs, &mut batch_offs),
+                    #[cfg(target_arch = "x86_64")]
+                    (2, true) => unsafe {
+                        merge_window_2_avx2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    #[cfg(target_arch = "x86_64")]
+                    (3, true) => unsafe {
+                        merge_window_3_avx2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    (2, _) => unsafe {
+                        merge_window_2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    (3, _) => unsafe {
+                        merge_window_3(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    _ => merge_window_n(
+                        &wins,
+                        &freq_bound_lut,
+                        others_block_max,
+                        kernel_threshold,
+                        &mut cursor_scratch,
+                        &mut batch_docs,
+                        &mut batch_offs,
+                    ),
+                }
+
+                // Pass A: gather doc norms/lengths for the whole batch up
+                // front so the loads issue back-to-back and their cache
+                // misses overlap. With the norm cache, only the byte code is
+                // gathered; the exact length decodes from a tiny LUT.
+                batch_lens.clear();
+                batch_norms.clear();
+                match norm_k_ref {
+                    Some((norms, _)) => {
+                        for &doc in batch_docs.iter() {
+                            batch_norms.push(norms[doc as usize]);
+                        }
+                    }
+                    None => match self.docs.scoring_norms() {
+                        Some(norms) => {
+                            for &doc in batch_docs.iter() {
+                                batch_lens.push(dequantize_doc_length(norms[doc as usize]));
+                            }
+                        }
+                        None => {
+                            for &doc in batch_docs.iter() {
+                                batch_lens.push(self.docs.scoring_num_tokens(doc));
+                            }
+                        }
+                    },
+                }
+
+                // Pass B: prune / verify / score / insert, in doc order, with
+                // exactly the classic loop's semantics.
+                for (index, &doc) in batch_docs.iter().enumerate() {
+                    let (norm_addend, doc_length) = match norm_k_ref {
+                        Some((_, cache)) => {
+                            let code = batch_norms[index];
+                            (Some(cache[code as usize]), dequantize_doc_length(code))
+                        }
+                        None => (None, batch_lens[index]),
+                    };
+                    let offs = &batch_offs[index * num_lists..(index + 1) * num_lists];
+                    if self.threshold > 0.0 && num_lists >= 2 {
+                        let first_freq = unsafe { *wins[0].freqs.add(offs[0] as usize) };
+                        let first_score = match norm_addend {
+                            Some(addend) => {
+                                self.lead[0].query_weight
+                                    * bm25_doc_weight_with_norm(first_freq, addend)
+                            }
+                            None => self.lead[0].score(&self.scorer, first_freq, doc_length),
+                        };
+                        if first_score + others_block_max <= self.threshold {
+                            self.and_candidates_pruned_before_return += 1;
+                            continue;
+                        }
+                    }
+                    stats.candidates_seen += 1;
+                    self.and_window_stats.candidates_returned += 1;
+                    num_comparisons += 1;
+
+                    let row_id = if docs_has_row_ids {
+                        self.docs.row_id(doc)
+                    } else {
+                        u64::from(doc)
+                    };
+                    if docs_has_row_ids
+                        && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id))
+                    {
+                        continue;
+                    }
+
+                    if let Some(slop) = phrase_slop {
+                        // Park every clause's iterator on this doc so
+                        // `position_cursor` reads the right posting entry. The
+                        // window block is already decompressed; position blocks
+                        // decode lazily and are cached per block.
+                        for ((win, posting), &off) in
+                            wins.iter().zip(self.lead.iter_mut()).zip(offs.iter())
+                        {
+                            posting.index = win.block_start + off as usize;
+                            posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id: doc,
+                                frequency: unsafe { *win.freqs.add(off as usize) },
+                            }));
+                        }
+                        let matched = if slop == 0 {
+                            self.check_exact_positions_bulk()?
+                        } else {
+                            self.check_positions(slop as i32)?
+                        };
+                        if !matched {
+                            continue;
+                        }
+                    }
+                    stats.full_scores += 1;
+
+                    let mut score = 0.0f32;
+                    for ((win, posting), &off) in wins.iter().zip(self.lead.iter()).zip(offs.iter())
+                    {
+                        let freq = unsafe { *win.freqs.add(off as usize) };
+                        score += match norm_addend {
+                            Some(addend) => {
+                                posting.query_weight * bm25_doc_weight_with_norm(freq, addend)
+                            }
+                            None => posting.score(&self.scorer, freq, doc_length),
+                        };
+                    }
+
+                    if candidates.insert(
+                        ScoredDoc::new(row_id, score),
+                        doc_length,
+                        u64::from(doc),
+                        wins.iter().zip(self.lead.iter()).zip(offs.iter()).map(
+                            |((win, posting), &off)| {
+                                (posting.term_index(), unsafe {
+                                    *win.freqs.add(off as usize)
+                                })
+                            },
+                        ),
+                    )? {
+                        stats.freqs_collected += 1;
+                        if let Some(kth) = candidates.kth_score_if_full() {
+                            self.update_threshold(kth, params.wand_factor);
+                        }
+                    }
+                }
+            }
+
+            if win_end == TERMINATED_DOC_ID {
+                break;
+            }
+            target = win_end + 1;
+        }
+
+        tracing::debug!(
+            and_windows_wide = self.and_window_stats.windows_wide,
+            and_windows_narrow = self.and_window_stats.windows_narrow,
+            and_windows_skipped = self.and_window_stats.windows_skipped,
+            and_range_blocks_scanned = self.and_window_stats.range_blocks_scanned,
+            and_candidates_returned = self.and_window_stats.candidates_returned,
+            "fts conjunction block-max window stats (bulk)"
+        );
+        metrics.record_comparisons(num_comparisons);
+        let pruned_before_return = self
+            .and_candidates_pruned_before_return
+            .saturating_sub(stats.pruned_before_return_start);
+        metrics.record_and_candidates_seen(stats.candidates_seen);
+        metrics.record_and_candidates_pruned_before_return(pruned_before_return);
+        metrics.record_and_full_scores(stats.full_scores);
+        metrics.record_freqs_collected(stats.freqs_collected);
+
+        let to_addr = |row_id_slot: u64| {
+            if docs_has_row_ids {
+                CandidateAddr::RowId(row_id_slot)
+            } else {
+                CandidateAddr::Pending(row_id_slot as u32)
+            }
+        };
+        candidates.into_candidates(to_addr)
+    }
+
     fn and_move_to_next_block(&mut self, target: u64) {
         if self.threshold <= 0.0 {
             self.up_to = Some(target);
@@ -1361,7 +3409,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let narrow_max_score = self
             .lead
             .iter()
-            .map(|posting| posting.block_max_score())
+            .map(|posting| posting.block_max_score(&self.scorer))
             .sum::<f32>();
 
         if narrow_max_score >= self.threshold {
@@ -1384,7 +3432,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             let mut wide_max_score = 0.0;
             let mut range_blocks_scanned = 0;
             for posting in &mut self.lead {
-                let block_max = posting.block_max_score_up_to_with_stats(lead_up_to);
+                let block_max = posting.block_max_score_up_to_with_stats(lead_up_to, &self.scorer);
                 wide_max_score += block_max.score;
                 range_blocks_scanned += block_max.blocks_scanned;
             }
@@ -1446,9 +3494,19 @@ impl<'a, S: Scorer> Wand<'a, S> {
     // Move all head iterators that are already known to be behind `target`
     // into `tail`, possibly overflowing low-value entries back into `head`.
     fn move_head_before_target_to_tail(&mut self, target: u64) {
+        if self.threshold <= 0.0 {
+            while matches!(self.head_doc(), Some(doc_id) if doc_id < target) {
+                if let Some(mut posting) = self.head.pop().map(|posting| posting.posting) {
+                    posting.next(target);
+                    self.push_head(posting);
+                }
+            }
+            return;
+        }
+
         while matches!(self.head_doc(), Some(doc_id) if doc_id < target) {
             if let Some(posting) = self.head.pop() {
-                let upper_bound = posting.posting.approximate_upper_bound();
+                let upper_bound = posting.posting.global_upper_bound(&self.scorer);
                 if let Some(mut evicted) =
                     self.insert_tail_with_overflow(posting.posting, upper_bound)
                 {
@@ -1467,12 +3525,12 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let lead: f32 = self
             .lead
             .iter()
-            .map(|posting| posting.block_max_score())
+            .map(|posting| posting.window_max_score(self.up_to, &self.scorer))
             .sum();
         let head: f32 = self
             .head
             .iter()
-            .map(|posting| posting.posting.block_max_score())
+            .map(|posting| posting.posting.window_max_score(self.up_to, &self.scorer))
             .sum();
         lead + head + self.tail_max_score
     }
@@ -1485,12 +3543,12 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let mut sum = self
             .lead
             .iter()
-            .map(|posting| posting.block_max_score())
+            .map(|posting| posting.window_max_score(self.up_to, &self.scorer))
             .sum::<f32>();
         let mut possible_matches = self.lead.len();
         for posting in &self.tail {
             if matches!(posting.posting.block_first_doc(), Some(block_doc) if block_doc <= target) {
-                sum += posting.posting.block_max_score();
+                sum += posting.posting.window_max_score(self.up_to, &self.scorer);
                 possible_matches += 1;
             }
         }
@@ -1504,72 +3562,121 @@ impl<'a, S: Scorer> Wand<'a, S> {
     fn update_max_scores(&mut self, target: u64) {
         // Refresh the block-max window for the current target. The resulting
         // `up_to` is the furthest doc id for which this block-max view remains
-        // valid.
+        // valid. Like Lucene's WANDScorer, the boundary comes from the cheap
+        // clauses only, and the refresh avoids allocating: heaps are recycled
+        // through their backing vectors (shallow_next never changes the doc a
+        // head entry is ordered by, so heapify restores the same shape).
         let lead_cost = self
             .lead
             .iter()
             .map(|posting| posting.cost())
             .min()
             .unwrap_or(usize::MAX);
-        let mut up_to = TERMINATED_DOC_ID;
+        let mut narrow_up_to = TERMINATED_DOC_ID;
         for posting in &mut self.lead {
             posting.shallow_next(target);
-            let block_end = posting
-                .next_block_first_doc()
-                .map(|doc| doc.saturating_sub(1))
-                .unwrap_or(TERMINATED_DOC_ID);
-            up_to = up_to.min(block_end);
-        }
-        let head = std::mem::take(&mut self.head);
-        let mut rebuilt_head = BinaryHeap::with_capacity(head.len());
-        for mut posting in head.into_vec() {
-            if posting.posting.cost() <= lead_cost {
-                posting.posting.shallow_next(posting.doc_id());
-                let block_end = posting
-                    .posting
-                    .next_block_first_doc()
-                    .map(|doc| doc.saturating_sub(1))
-                    .unwrap_or(TERMINATED_DOC_ID);
-                up_to = up_to.min(block_end);
-            }
-            rebuilt_head.push(posting);
+            narrow_up_to = narrow_up_to.min(posting.block_end_doc());
+        }
+
+        let mut head_postings = std::mem::take(&mut self.head).into_vec();
+        for posting in &mut head_postings {
+            // Unlike Lucene, every head clause participates in the boundary:
+            // the refresh is allocation-free and answers from the anchored
+            // level caches, so frequent refreshes are cheap, while keeping the
+            // window inside every clause's current block keeps all the bounds
+            // at tight level0 values.
+            let doc_id = posting.doc_id();
+            posting.posting.shallow_next(doc_id);
+            narrow_up_to = narrow_up_to.min(posting.posting.block_end_doc());
+        }
+
+        let mut tail_postings = std::mem::take(&mut self.tail).into_vec();
+        for tail_posting in &mut tail_postings {
+            tail_posting.posting.shallow_next(target);
         }
-        self.head = rebuilt_head;
-        if up_to == TERMINATED_DOC_ID
-            && let Some(top) = self.tail.peek()
-            && top.cost <= lead_cost
+
+        if narrow_up_to == TERMINATED_DOC_ID
+            && let Some(top) = tail_postings
+                .iter()
+                .min_by_key(|posting| posting.posting.cost())
+            && top.posting.cost() <= lead_cost
         {
-            let block_end = top
-                .posting
-                .next_block_first_doc()
-                .map(|doc| doc.saturating_sub(1))
-                .unwrap_or(TERMINATED_DOC_ID);
-            up_to = up_to.min(block_end.max(target));
+            narrow_up_to = narrow_up_to.min(top.posting.block_end_doc().max(target));
         }
-        self.up_to = Some(up_to);
 
-        let tail = std::mem::take(&mut self.tail);
+        self.up_to = Some(narrow_up_to);
+        self.head = BinaryHeap::from(head_postings);
+
         self.tail_max_score = 0.0;
-        for mut tail_posting in tail.into_vec() {
-            tail_posting.posting.shallow_next(target);
-            let upper_bound = match tail_posting.posting.block_first_doc() {
+        for tail_posting in tail_postings {
+            let posting = tail_posting.posting;
+            let upper_bound = match posting.block_first_doc() {
                 Some(block_doc) if block_doc <= target => {
-                    tail_posting
-                        .posting
-                        .block_max_score_up_to_with_stats(up_to)
-                        .score
+                    posting.window_max_score(self.up_to, &self.scorer)
                 }
                 _ => 0.0,
             };
-            if let Some(mut evicted) =
-                self.insert_tail_with_overflow(tail_posting.posting, upper_bound)
-            {
+            if let Some(mut evicted) = self.insert_tail_with_overflow(posting, upper_bound) {
                 evicted.next(target);
                 self.push_head(evicted);
             }
         }
     }
 
+    /// After the narrow window proved skippable, try widening the skip to the
+    /// level1 group boundary, in the spirit of Lucene's `getSkipUpTo`. All
+    /// bounds come from the anchored level caches, so a failed attempt costs a
+    /// few loads and float adds — unlike an eager wide-window probe.
+    ///
+    /// The group boundary is the minimum current-group end over every live
+    /// iterator, so each iterator's level1 score is a valid bound over
+    /// `[target, group_up_to]`.
+    fn or_group_skip_to(&self) -> Option<u64> {
+        let mut group_up_to = TERMINATED_DOC_ID;
+        for posting in &self.lead {
+            let (doc_up_to, _) = posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        for posting in self.head.iter() {
+            let (doc_up_to, _) = posting.posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        for tail_posting in self.tail.iter() {
+            let (doc_up_to, _) = tail_posting.posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        if self.up_to.is_some_and(|up_to| group_up_to <= up_to) {
+            // No gain over the narrow window skip.
+            return None;
+        }
+
+        // Second pass over the memoized bounds: sum only the iterators that
+        // can produce a doc inside the skipped range.
+        let mut bounds_sum = 0.0_f32;
+        for posting in &self.lead {
+            let (_, score) = posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        for posting in self.head.iter() {
+            if posting.doc_id() > group_up_to {
+                continue;
+            }
+            let (_, score) = posting.posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        for tail_posting in self.tail.iter() {
+            if !matches!(
+                tail_posting.posting.block_first_doc(),
+                Some(block_doc) if block_doc <= group_up_to
+            ) {
+                continue;
+            }
+            let (_, score) = tail_posting.posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        (bounds_sum <= self.threshold).then_some(group_up_to.saturating_add(1))
+    }
+
     fn refine_or_candidate(&mut self, target: u64, doc_length: u32) -> bool {
         if self.threshold <= 0.0 {
             return true;
@@ -1693,9 +3800,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
             && posting.is_compressed()
             && self.up_to.is_some_and(|up_to| target <= up_to)
         {
-            posting.block_max_score()
+            posting.window_max_score(self.up_to, &self.scorer)
         } else {
-            posting.approximate_upper_bound()
+            posting.global_upper_bound(&self.scorer)
         }
     }
 
@@ -1730,6 +3837,14 @@ impl<'a, S: Scorer> Wand<'a, S> {
         // into lagging iterators. Entries that do not stay in `tail` are
         // advanced to `target` and returned to `head`.
         // pop() drains in place, keeping self.lead's capacity for reuse.
+        if self.threshold <= 0.0 {
+            while let Some(mut posting) = self.lead.pop() {
+                posting.next(target);
+                self.push_head(posting);
+            }
+            return;
+        }
+
         while let Some(posting) = self.lead.pop() {
             let upper_bound = self.lead_to_tail_upper_bound(&posting, target);
             if let Some(mut evicted) = self.insert_tail_with_overflow(posting, upper_bound) {
@@ -1806,7 +3921,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             .collect()
     }
 
-    fn check_positions(&self, slop: i32) -> bool {
+    fn check_positions(&self, slop: i32) -> Result<bool> {
         if slop == 0 {
             return self.check_exact_positions();
         }
@@ -1814,8 +3929,8 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let mut position_iters = self
             .current_doc_postings()
             .into_iter()
-            .map(|posting| posting.position_cursor().expect("positions must exist"))
-            .collect::<Vec<_>>();
+            .map(PostingIterator::position_cursor)
+            .collect::<Result<Vec<_>>>()?;
         position_iters.sort_unstable_by_key(|iter| iter.position_in_query);
 
         loop {
@@ -1825,7 +3940,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 let last = window[0].relative_position();
                 let next = window[1].relative_position();
                 let (Some(last), Some(next)) = (last, next) else {
-                    return false;
+                    return Ok(false);
                 };
 
                 let move_to = if last > next {
@@ -1841,7 +3956,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             if all_same {
-                return true;
+                return Ok(true);
             }
 
             position_iters.iter_mut().for_each(|iter| {
@@ -1850,21 +3965,72 @@ impl<'a, S: Scorer> Wand<'a, S> {
         }
     }
 
-    fn check_exact_positions(&self) -> bool {
+    /// Allocation-free exact-phrase check for the bulk conjunction path,
+    /// where every clause is a parked `lead` iterator. Semantically identical
+    /// to [`Self::check_exact_positions`] — some base position must align all
+    /// clauses at their query offsets — without the per-candidate cursor vec
+    /// and sort.
+    fn check_exact_positions_bulk(&self) -> Result<bool> {
+        const MAX_INLINE_CLAUSES: usize = 16;
+        let num_clauses = self.lead.len();
+        if num_clauses > MAX_INLINE_CLAUSES {
+            return self.check_exact_positions();
+        }
+        // Cursors stay alive in the stack array so owned position buffers
+        // (legacy per-doc storage) remain valid while we scan.
+        let mut cursors: [Option<PositionCursor<'_>>; MAX_INLINE_CLAUSES] =
+            std::array::from_fn(|_| None);
+        let mut anchor_idx = 0usize;
+        let mut anchor_len = usize::MAX;
+        for (index, (slot, posting)) in cursors.iter_mut().zip(self.lead.iter()).enumerate() {
+            let cursor = posting.position_cursor()?;
+            if cursor.len() < anchor_len {
+                anchor_len = cursor.len();
+                anchor_idx = index;
+            }
+            *slot = Some(cursor);
+        }
+
+        let anchor = cursors[anchor_idx]
+            .as_ref()
+            .expect("anchor cursor was just populated");
+        let anchor_offset = anchor.position_in_query as u32;
+        'anchor: for &anchor_position in anchor.positions.as_slice() {
+            let Some(base) = anchor_position.checked_sub(anchor_offset) else {
+                continue;
+            };
+            for (index, slot) in cursors[..num_clauses].iter().enumerate() {
+                if index == anchor_idx {
+                    continue;
+                }
+                let cursor = slot.as_ref().expect("clause cursor was just populated");
+                let Some(target) = base.checked_add(cursor.position_in_query as u32) else {
+                    return Ok(false);
+                };
+                if cursor.positions.as_slice().binary_search(&target).is_err() {
+                    continue 'anchor;
+                }
+            }
+            return Ok(true);
+        }
+        Ok(false)
+    }
+
+    fn check_exact_positions(&self) -> Result<bool> {
         let mut position_iters = self
             .current_doc_postings()
             .into_iter()
-            .map(|posting| posting.position_cursor().expect("positions must exist"))
-            .collect::<Vec<_>>();
+            .map(PostingIterator::position_cursor)
+            .collect::<Result<Vec<_>>>()?;
         position_iters.sort_unstable_by_key(|iter| iter.len());
         let Some(lead) = position_iters.first() else {
-            return false;
+            return Ok(false);
         };
         let lead_position = lead.position_in_query;
 
         loop {
             let Some(anchor) = position_iters[0].absolute_position() else {
-                return false;
+                return Ok(false);
             };
             let Some(base) = anchor.checked_sub(lead_position as u32) else {
                 position_iters[0].advance_next();
@@ -1875,10 +4041,10 @@ impl<'a, S: Scorer> Wand<'a, S> {
             let mut matched = true;
             for follower in position_iters.iter_mut().skip(1) {
                 let Some(target) = base.checked_add(follower.position_in_query as u32) else {
-                    return false;
+                    return Ok(false);
                 };
                 let Some(position) = follower.advance_to_absolute(target) else {
-                    return false;
+                    return Ok(false);
                 };
                 if position != target {
                     next_lead_relative = Some(position as i32 - follower.position_in_query);
@@ -1888,7 +4054,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             if matched {
-                return true;
+                return Ok(true);
             }
 
             position_iters[0].advance_to_relative(next_lead_relative.unwrap());
@@ -1896,16 +4062,50 @@ impl<'a, S: Scorer> Wand<'a, S> {
     }
 }
 
+#[derive(Debug)]
+struct RecycledPositionValues<'a> {
+    values: Option<Vec<u32>>,
+    pool: &'a RefCell<Option<Vec<u32>>>,
+}
+
+impl<'a> RecycledPositionValues<'a> {
+    fn new(values: Vec<u32>, pool: &'a RefCell<Option<Vec<u32>>>) -> Self {
+        Self {
+            values: Some(values),
+            pool,
+        }
+    }
+
+    fn as_slice(&self) -> &[u32] {
+        self.values
+            .as_deref()
+            .expect("position values are present until drop")
+    }
+}
+
+impl Drop for RecycledPositionValues<'_> {
+    fn drop(&mut self) {
+        let values = self
+            .values
+            .take()
+            .expect("position values are present until drop");
+        let mut pool = self.pool.borrow_mut();
+        if pool.is_none() {
+            *pool = Some(values);
+        }
+    }
+}
+
 #[derive(Debug)]
 enum PositionValues<'a> {
-    Borrowed(&'a [u32]),
+    Recycled(RecycledPositionValues<'a>),
     Owned(Vec<u32>),
 }
 
 impl<'a> PositionValues<'a> {
     fn as_slice(&self) -> &[u32] {
         match self {
-            Self::Borrowed(values) => values,
+            Self::Recycled(values) => values.as_slice(),
             Self::Owned(values) => values.as_slice(),
         }
     }
@@ -1977,19 +4177,61 @@ mod tests {
 
     use std::sync::atomic::{AtomicUsize, Ordering};
 
+    use super::super::impact::build_impact_skip_data;
     use super::*;
-    use crate::scalar::inverted::scorer::IndexBM25Scorer;
+    use crate::scalar::inverted::scorer::{IndexBM25Scorer, MemBM25Scorer};
     use crate::{
         metrics::{MetricsCollector, NoOpMetricsCollector},
         scalar::inverted::{
-            CompressedPostingList, PlainPostingList, PostingListBuilder, builder::PositionRecorder,
-            encoding::compress_posting_list,
+            CompressedPostingList, PlainPostingList, PostingListBuilder, SharedPositionStream,
+            builder::PositionRecorder,
+            encoding::{
+                compress_posting_list, compress_posting_list_with_tail_codec_and_block_size,
+                encode_position_stream_block_into,
+            },
         },
     };
 
+    #[test]
+    fn test_maxscore_prefix_bound_covers_f32_summation_rounding() {
+        let remaining_bounds = [6.286_838_4e-7_f32, 0.015_441_144_f32];
+        let essential_score = 2.762_496_2_f32;
+        let threshold = f32::from_bits(0x4031_c9bc);
+
+        // Raw f32 prefix accumulation rounds down to an apparent tie, while
+        // scoring the clauses individually produces a competitive document.
+        let raw_prefix = remaining_bounds.into_iter().sum::<f32>();
+        assert_eq!(essential_score + raw_prefix, threshold);
+        let actual_score = remaining_bounds
+            .into_iter()
+            .rev()
+            .fold(essential_score, |score, bound| score + bound);
+        assert!(actual_score > threshold);
+
+        let prefix_bound = remaining_bounds.into_iter().map(f64::from).sum::<f64>();
+        assert!(!score_sum_cannot_exceed(
+            essential_score,
+            prefix_bound,
+            threshold,
+            score_sum_upper_bound_factor(3),
+        ));
+    }
+
     struct UnitScorer;
 
-    impl Scorer for UnitScorer {
+    impl Scorer for UnitScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, _doc_tokens: u32) -> f32 {
+            freq as f32
+        }
+    }
+
+    struct PartialNormScorer;
+
+    impl Scorer for PartialNormScorer {
         fn query_weight(&self, _token: &str) -> f32 {
             1.0
         }
@@ -1997,6 +4239,86 @@ mod tests {
         fn doc_weight(&self, freq: u32, _doc_tokens: u32) -> f32 {
             freq as f32
         }
+
+        fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+            (doc_tokens == 0).then_some(0.0)
+        }
+    }
+
+    #[test]
+    fn test_norm_cache_requires_all_norm_codes() {
+        let mut docs = DocSet::default();
+        docs.append(0, 1);
+        docs.set_quantized_scoring(true);
+        let wand = Wand::new(Operator::Or, std::iter::empty(), &docs, PartialNormScorer);
+
+        assert!(wand.norm_k_cache().is_none());
+    }
+
+    #[test]
+    fn test_top_k_collector_reuses_frequency_slots() -> Result<()> {
+        const LIMIT: usize = 8;
+        const NUM_DOCS: usize = 10_000;
+        let mut collector = TopKCollector::new(LIMIT, LIMIT);
+
+        for doc in 0..NUM_DOCS {
+            let num_terms = doc % 4 + 1;
+            let inserted = collector.insert(
+                ScoredDoc::new(doc as u64, doc as f32),
+                num_terms as u32,
+                doc as u64,
+                (0..num_terms).map(|term| (term as u32, doc as u32)),
+            )?;
+            assert!(inserted);
+            assert!(collector.num_frequency_slots() <= LIMIT);
+        }
+        assert_eq!(collector.num_frequency_slots(), LIMIT);
+
+        let mut candidates = collector.into_candidates(CandidateAddr::RowId)?;
+        candidates.sort_unstable_by_key(|candidate| candidate.posting_doc_id);
+        assert_eq!(candidates.len(), LIMIT);
+        for (candidate, expected_doc) in candidates.iter().zip(NUM_DOCS - LIMIT..NUM_DOCS) {
+            assert_eq!(candidate.posting_doc_id, expected_doc as u64);
+            assert!(matches!(
+                candidate.addr,
+                CandidateAddr::RowId(row_id) if row_id == expected_doc as u64
+            ));
+            let expected_freqs = (0..expected_doc % 4 + 1)
+                .map(|term| (term as u32, expected_doc as u32))
+                .collect::<Vec<_>>();
+            assert_eq!(candidate.freqs, expected_freqs);
+        }
+        Ok(())
+    }
+
+    #[rstest]
+    #[case::auto("auto", Some(BulkAndMode::Auto))]
+    #[case::auto_case_and_whitespace(" AUTO ", Some(BulkAndMode::Auto))]
+    #[case::on("on", Some(BulkAndMode::On))]
+    #[case::on_legacy("1", Some(BulkAndMode::On))]
+    #[case::off("off", Some(BulkAndMode::Off))]
+    #[case::off_legacy("0", Some(BulkAndMode::Off))]
+    #[case::invalid("true", None)]
+    #[case::empty("", None)]
+    fn test_bulk_and_mode_parse(#[case] value: &str, #[case] expected: Option<BulkAndMode>) {
+        assert_eq!(BulkAndMode::parse(value), expected);
+    }
+
+    #[rstest]
+    #[case::auto_one(BulkAndMode::Auto, 1, false)]
+    #[case::auto_two(BulkAndMode::Auto, 2, true)]
+    #[case::auto_three(BulkAndMode::Auto, 3, true)]
+    #[case::auto_four(BulkAndMode::Auto, 4, false)]
+    #[case::on_one(BulkAndMode::On, 1, true)]
+    #[case::on_five(BulkAndMode::On, 5, true)]
+    #[case::off_two(BulkAndMode::Off, 2, false)]
+    #[case::off_five(BulkAndMode::Off, 5, false)]
+    fn test_bulk_and_mode_enabled_for(
+        #[case] mode: BulkAndMode,
+        #[case] num_clauses: usize,
+        #[case] expected: bool,
+    ) {
+        assert_eq!(mode.enabled_for(num_clauses), expected);
     }
 
     struct PanicQueryWeightScorer;
@@ -2146,6 +4468,8 @@ mod tests {
                 max_score,
                 doc_ids.len() as u32,
                 crate::scalar::inverted::PostingTailCodec::VarintDelta,
+                crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+                None,
                 None,
             ))
         } else {
@@ -2158,6 +4482,75 @@ mod tests {
         }
     }
 
+    fn generate_impact_posting_list_with_freqs(
+        doc_ids: Vec<u32>,
+        freqs: Vec<u32>,
+        doc_lengths: Vec<u32>,
+    ) -> PostingList {
+        generate_impact_posting_list_with_freqs_and_block_size(
+            doc_ids,
+            freqs,
+            doc_lengths,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    fn generate_impact_posting_list_with_freqs_and_block_size(
+        doc_ids: Vec<u32>,
+        freqs: Vec<u32>,
+        doc_lengths: Vec<u32>,
+        block_size: usize,
+    ) -> PostingList {
+        assert_eq!(doc_ids.len(), freqs.len());
+        assert_eq!(doc_ids.len(), doc_lengths.len());
+        let block_max_scores = vec![0.0; doc_ids.len().div_ceil(block_size)];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            freqs.iter(),
+            block_max_scores.into_iter(),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            block_size,
+        )
+        .unwrap();
+        let impact_blocks = doc_ids
+            .chunks(block_size)
+            .zip(freqs.chunks(block_size))
+            .zip(doc_lengths.chunks(block_size))
+            .map(|((doc_ids, freqs), doc_lengths)| {
+                doc_ids
+                    .iter()
+                    .copied()
+                    .zip(freqs.iter().copied())
+                    .zip(doc_lengths.iter().copied())
+                    .map(|((doc_id, freq), doc_length)| (doc_id, freq, doc_length))
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(impact_blocks.as_slice()).unwrap();
+        PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            0.0,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            block_size,
+            None,
+            Some(impacts),
+        ))
+    }
+
+    fn generate_contiguous_impact_posting_list_with_block_size(
+        total: usize,
+        block_size: usize,
+    ) -> PostingList {
+        generate_impact_posting_list_with_freqs_and_block_size(
+            (0..total as u32).collect(),
+            vec![1; total],
+            vec![1; total],
+            block_size,
+        )
+    }
+
     fn generate_posting_list_with_positions(
         doc_ids: Vec<u32>,
         positions_by_doc: Vec<Vec<u32>>,
@@ -2195,6 +4588,95 @@ mod tests {
         }
     }
 
+    #[rstest]
+    #[case::packed_delta(PositionStreamCodec::PackedDelta)]
+    #[case::varint_doc_delta(PositionStreamCodec::VarintDocDelta)]
+    fn test_shared_position_cursors_use_independent_scratch(
+        #[case] codec: PositionStreamCodec,
+    ) -> Result<()> {
+        let mut posting_list =
+            generate_posting_list_with_positions(vec![0], vec![vec![1_u32, 3, 10]], 1.0, true);
+        let PostingList::Compressed(ref mut list) = posting_list else {
+            unreachable!("the helper was asked for a compressed posting list");
+        };
+        let mut encoded = Vec::new();
+        encode_position_stream_block_into(&[1, 3, 10], &[3], codec, &mut encoded)?;
+        list.positions = Some(CompressedPositionStorage::SharedStream(
+            SharedPositionStream::new(codec, vec![0], bytes::Bytes::from(encoded)),
+        ));
+
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, 1);
+        let first = posting.position_cursor()?;
+        let second = posting.position_cursor()?;
+
+        assert_eq!(second.positions.as_slice(), &[1, 3, 10]);
+        assert_eq!(first.positions.as_slice(), &[1, 3, 10]);
+        assert!(posting.position_scratch.borrow().is_none());
+        drop(second);
+        assert!(posting.position_scratch.borrow().is_some());
+        drop(first);
+        assert!(posting.position_scratch.borrow().is_some());
+        Ok(())
+    }
+
+    #[test]
+    fn test_phrase_search_propagates_corrupt_packed_positions() {
+        let mut docs = DocSet::default();
+        docs.append(0, BLOCK_SIZE as u32 + 1);
+
+        let mut corrupt_list = generate_posting_list_with_positions(
+            vec![0],
+            vec![(0..BLOCK_SIZE as u32).collect()],
+            1.0,
+            true,
+        );
+        let PostingList::Compressed(ref mut list) = corrupt_list else {
+            unreachable!("the helper was asked for a compressed posting list");
+        };
+        // A bit width of one requires a 16-byte payload. Keep only the header
+        // to verify malformed on-disk data becomes a search error, not a panic.
+        list.positions = Some(CompressedPositionStorage::SharedStream(
+            SharedPositionStream::new(
+                PositionStreamCodec::PackedDelta,
+                vec![0],
+                bytes::Bytes::from_static(&[1]),
+            ),
+        ));
+
+        let postings = vec![
+            PostingIterator::new(String::from("corrupt"), 0, 0, corrupt_list, docs.len()),
+            PostingIterator::new(
+                String::from("valid"),
+                1,
+                1,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    vec![vec![BLOCK_SIZE as u32]],
+                    1.0,
+                    true,
+                ),
+                docs.len(),
+            ),
+        ];
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        let mut params = FtsSearchParams::default().with_limit(Some(10));
+        params.phrase_slop = Some(0);
+
+        let error = wand
+            .search(
+                &params,
+                Arc::new(RowAddrMask::default()),
+                &NoOpMetricsCollector,
+            )
+            .expect_err("corrupt packed positions should fail the phrase search");
+        let message = error.to_string();
+        assert!(
+            message.contains("packed position group payload"),
+            "{message}"
+        );
+        assert!(message.contains("corrupt"), "{message}");
+    }
+
     fn sorted_candidate_row_ids(candidates: Vec<DocCandidate>) -> Vec<u64> {
         let mut row_ids = candidates
             .into_iter()
@@ -2652,6 +5134,56 @@ mod tests {
         assert_eq!(candidate.0.doc_id(), BLOCK_SIZE as u64);
     }
 
+    #[test]
+    fn test_non_positive_threshold_advances_without_impact_bound_scoring() {
+        let mut docs = DocSet::default();
+        for doc_id in 0..3 {
+            docs.append(doc_id, 1);
+        }
+        let make_posting = || {
+            PostingIterator::with_query_weight(
+                String::from("term"),
+                0,
+                0,
+                1.0,
+                generate_impact_posting_list_with_freqs(
+                    vec![0, 1, 2],
+                    vec![1, 1, 1],
+                    vec![1, 1, 1],
+                ),
+                docs.len(),
+            )
+        };
+        let scored = Arc::new(AtomicUsize::new(0));
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(make_posting()),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.move_head_before_target_to_tail(1);
+        assert_eq!(wand.head_doc(), Some(1));
+        assert!(wand.tail.is_empty());
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(make_posting()),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.move_head_doc_to_lead(0);
+        wand.push_back_leads(1);
+        assert_eq!(wand.head_doc(), Some(1));
+        assert!(wand.tail.is_empty());
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+    }
+
     #[test]
     fn test_or_plain_tail_does_not_advance_headless_window() {
         let mut docs = DocSet::default();
@@ -2982,7 +5514,7 @@ mod tests {
         posting.shallow_next(0);
         assert_eq!(
             posting
-                .block_max_score_up_to_with_stats((3 * BLOCK_SIZE - 1) as u64)
+                .block_max_score_up_to_with_stats((3 * BLOCK_SIZE - 1) as u64, &UnitScorer)
                 .score,
             4.0
         );
@@ -2990,7 +5522,7 @@ mod tests {
         posting.shallow_next((2 * BLOCK_SIZE) as u64);
         assert_eq!(
             posting
-                .block_max_score_up_to_with_stats((4 * BLOCK_SIZE - 1) as u64)
+                .block_max_score_up_to_with_stats((4 * BLOCK_SIZE - 1) as u64, &UnitScorer)
                 .score,
             5.0
         );
@@ -2998,12 +5530,219 @@ mod tests {
         posting.shallow_next((4 * BLOCK_SIZE) as u64);
         assert_eq!(
             posting
-                .block_max_score_up_to_with_stats((5 * BLOCK_SIZE - 1) as u64)
+                .block_max_score_up_to_with_stats((5 * BLOCK_SIZE - 1) as u64, &UnitScorer)
                 .score,
             3.0
         );
     }
 
+    #[test]
+    fn test_impact_level1_skip_keeps_boundary_equality_in_group() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS + 1) * block_size;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+            let target = (IMPACT_LEVEL1_BLOCKS * block_size - 1) as u64;
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, IMPACT_LEVEL1_BLOCKS - 1);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, IMPACT_LEVEL1_BLOCKS - 1);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_impact_level1_skip_handles_partial_final_group() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS + 3) * block_size + 17;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+            let target = (total - 1) as u64;
+            let expected_block = total.div_ceil(block_size) - 1;
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, expected_block);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, expected_block);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_impact_level1_skip_reaches_far_target_doc() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS * 3 + 5) * block_size;
+            let target_block = IMPACT_LEVEL1_BLOCKS * 2 + 2;
+            let target = (target_block * block_size + 17) as u64;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, target_block);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, target_block);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_or_impact_level1_window_skips_low_group_with_single_score() {
+        let total = (IMPACT_LEVEL1_BLOCKS + 1) * BLOCK_SIZE;
+        let target = (IMPACT_LEVEL1_BLOCKS * BLOCK_SIZE) as u64;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total as u64 {
+            docs.append(doc_id, 1);
+        }
+
+        let doc_ids = (0..total as u32).collect::<Vec<_>>();
+        let freqs = doc_ids
+            .iter()
+            .map(|doc_id| if u64::from(*doc_id) < target { 1 } else { 10 })
+            .collect::<Vec<_>>();
+        let posting_list = generate_impact_posting_list_with_freqs(doc_ids, freqs, vec![1; total]);
+        let mut probe = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            posting_list.clone(),
+            docs.len(),
+        );
+        probe.shallow_next(0);
+        let counting_scorer = CountingScorer {
+            scored: Arc::new(AtomicUsize::new(0)),
+        };
+        let (group_up_to, group_score) = probe.impact_group_bound(&counting_scorer).unwrap();
+        assert_eq!(group_up_to, target - 1);
+        assert_eq!(group_score, 1.0);
+        // A window ending past the current block must answer from level1.
+        assert_eq!(
+            probe.window_max_score(Some(target - 1), &counting_scorer),
+            1.0
+        );
+
+        let posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            posting_list,
+            docs.len(),
+        );
+        let scored = Arc::new(AtomicUsize::new(0));
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(posting),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.threshold = 2.0;
+
+        let (candidate, score) = wand.next().unwrap().unwrap();
+        assert_eq!(candidate.doc_id(), target);
+        assert_eq!(score, 10.0);
+        // The doc-weight bounds bake exactly once (one doc_weight call per
+        // frontier pair across all entries); beyond that only the returned
+        // candidate is scored.
+        let total_entries = (IMPACT_LEVEL1_BLOCKS + 1) + 2;
+        assert!(
+            scored.load(Ordering::Relaxed) <= total_entries + 8,
+            "bounds should be baked once instead of recomputed per window; scored={}",
+            scored.load(Ordering::Relaxed)
+        );
+    }
+
+    #[test]
+    fn test_compressed_impact_block_max_score_memoizes_current_block() {
+        let total = 2 * BLOCK_SIZE as u32;
+        let doc_ids = (0..total).collect::<Vec<_>>();
+        let freqs = doc_ids
+            .iter()
+            .map(|doc_id| if *doc_id < BLOCK_SIZE as u32 { 1 } else { 2 })
+            .collect::<Vec<_>>();
+        let doc_lengths = vec![1; total as usize];
+        let posting_list = generate_impact_posting_list_with_freqs(doc_ids, freqs, doc_lengths);
+        let mut posting =
+            PostingIterator::new(String::from("term"), 0, 0, posting_list, total as usize);
+        let scored = Arc::new(AtomicUsize::new(0));
+        let scorer = CountingScorer {
+            scored: scored.clone(),
+        };
+
+        let first_score = posting.block_max_score(&scorer);
+        assert_eq!(first_score, 1.0);
+        // Baking the query-local doc-weight bounds visits every frontier pair
+        // once (two level0 entries plus one level1 entry for this list).
+        let baked = scored.load(Ordering::Relaxed);
+        assert!(baked >= 2);
+        {
+            let compressed = unsafe { &mut *posting.compressed_state_ptr() };
+            assert_eq!(
+                compressed.level0_cache,
+                Some((0, BLOCK_SIZE as u32 - 1, first_score))
+            );
+        }
+
+        let second_score = posting.block_max_score(&scorer);
+        assert_eq!(second_score, first_score);
+        assert_eq!(
+            scored.load(Ordering::Relaxed),
+            baked,
+            "repeated block max scores must not recompute doc weights"
+        );
+
+        posting.shallow_next(BLOCK_SIZE as u64);
+        let next_block_score = posting.block_max_score(&scorer);
+        assert_eq!(next_block_score, 2.0);
+        assert_eq!(
+            scored.load(Ordering::Relaxed),
+            baked,
+            "other blocks answer from the baked bounds without rescoring"
+        );
+    }
+
+    #[rstest]
+    #[case(0.0)]
+    #[case(-1.0)]
+    fn test_non_positive_query_weight_skips_global_impact_bound(#[case] query_weight: f32) {
+        let posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            query_weight,
+            generate_impact_posting_list_with_freqs(vec![0], vec![1], vec![1]),
+            1,
+        );
+        let scored = Arc::new(AtomicUsize::new(0));
+        let scorer = CountingScorer {
+            scored: scored.clone(),
+        };
+
+        assert_eq!(posting.global_upper_bound(&scorer), 0.0);
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+    }
+
     #[test]
     fn test_and_candidate_prune_scores_first_term_before_full_score() {
         let total_docs = 2 * BLOCK_SIZE as u32 + 1;
@@ -3055,8 +5794,11 @@ mod tests {
         let addrs = result.into_iter().map(|doc| doc.addr).collect::<Vec<_>>();
         assert!(matches!(addrs.as_slice(), [CandidateAddr::RowId(0)]));
         let scored = scored.load(Ordering::Relaxed);
+        // The bulk path evaluates 63 doc weights up front to fill its
+        // frequency-bound prune LUT; those are bound computations, not
+        // per-candidate scoring.
         assert!(
-            scored <= BLOCK_SIZE + 1,
+            scored <= BLOCK_SIZE + 1 + 63,
             "expected candidate pruning to avoid full scoring in the first block, scored {scored}"
         );
     }
@@ -3340,13 +6082,93 @@ mod tests {
 
         let posting = PostingIterator::new(String::from("test"), 0, 0, posting_list, 1);
 
-        let actual = posting.block_max_score();
+        let actual = posting.block_max_score(&UnitScorer);
         assert!(
             (actual - expected).abs() < 1e-6,
             "block max score should match stored value"
         );
     }
 
+    #[test]
+    fn test_256_document_blocks_without_impacts_use_conservative_quantized_score_bound() {
+        let exact_doc_length = 300;
+        let quantized_doc_length = super::super::index::dequantize_doc_length(
+            super::super::index::quantize_doc_length(exact_doc_length),
+        );
+        assert!(quantized_doc_length < exact_doc_length);
+
+        let scorer = Arc::new(MemBM25Scorer::new(100, 1, Default::default()));
+        let stored_exact_score = scorer.doc_weight(1, exact_doc_length);
+        let quantized_score = scorer.doc_weight(1, quantized_doc_length);
+        assert!(quantized_score > stored_exact_score);
+
+        let doc_ids = [0_u32];
+        let frequencies = [1_u32];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(stored_exact_score),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+        )
+        .unwrap();
+        let posting_list = PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            stored_exact_score,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, doc_ids.len());
+        let expected_bound = K1 + 1.0;
+
+        assert_eq!(posting.approximate_upper_bound(), expected_bound);
+        assert_eq!(posting.global_upper_bound(&scorer), expected_bound);
+        assert_eq!(posting.block_max_score(&scorer), expected_bound);
+        assert_eq!(
+            posting.block_max_score_up_to_with_stats(0, &scorer).score,
+            expected_bound
+        );
+        assert!(expected_bound >= quantized_score);
+    }
+
+    #[test]
+    fn test_256_document_blocks_without_impacts_unknown_scorer_uses_infinite_bound() {
+        let doc_ids = [0_u32];
+        let frequencies = [10_u32];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(10.0),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+        )
+        .unwrap();
+        let posting_list = PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            10.0,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, doc_ids.len());
+
+        assert!(posting.global_upper_bound(&UnitScorer).is_infinite());
+        assert!(posting.block_max_score(&UnitScorer).is_infinite());
+        assert!(
+            posting
+                .block_max_score_up_to_with_stats(0, &UnitScorer)
+                .score
+                .is_infinite()
+        );
+    }
+
     #[rstest]
     fn test_exact_phrase_with_repeated_terms(#[values(false, true)] is_compressed: bool) {
         let mut docs = DocSet::default();
@@ -3395,8 +6217,8 @@ mod tests {
 
         let bm25 = IndexBM25Scorer::new(std::iter::empty());
         let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
-        assert!(wand.check_exact_positions());
-        assert!(wand.check_positions(0));
+        assert!(wand.check_exact_positions().unwrap());
+        assert!(wand.check_positions(0).unwrap());
     }
 
     #[rstest]
@@ -3433,8 +6255,8 @@ mod tests {
 
         let bm25 = IndexBM25Scorer::new(std::iter::empty());
         let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
-        assert!(wand.check_exact_positions());
-        assert!(wand.check_positions(0));
+        assert!(wand.check_exact_positions().unwrap());
+        assert!(wand.check_positions(0).unwrap());
     }
 
     #[rstest]
@@ -3473,11 +6295,149 @@ mod tests {
         let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
         let first = wand.next().unwrap().unwrap();
         assert_eq!(first.0.doc_id(), 0);
-        assert!(!wand.check_positions(0));
+        assert!(!wand.check_positions(0).unwrap());
 
         wand.threshold = 1.5;
         let second = wand.next().unwrap().unwrap();
         assert_eq!(second.0.doc_id(), 1);
-        assert!(wand.check_positions(0));
+        assert!(wand.check_positions(0).unwrap());
+    }
+
+    /// The bulk conjunction path must return exactly the classic loop's
+    /// results — same docs, freqs, and doc lengths — for both plain AND and
+    /// phrase queries, across multi-block lists with heap/threshold pruning
+    /// in play.
+    #[rstest]
+    #[case::and_k10(false, 0, 10, 3)]
+    #[case::and_k3(false, 0, 3, 3)]
+    #[case::and_two_clauses(false, 0, 10, 2)]
+    #[case::and_four_clauses(false, 0, 10, 4)]
+    #[case::and_five_clauses(false, 0, 10, 5)]
+    #[case::and_six_clauses(false, 0, 10, 6)]
+    #[case::phrase_k10(true, 0, 10, 3)]
+    #[case::phrase_k3(true, 0, 3, 3)]
+    #[case::phrase_slop_three(true, 3, 10, 3)]
+    #[case::phrase_two_clauses(true, 0, 10, 2)]
+    #[case::phrase_four_clauses(true, 0, 10, 4)]
+    #[case::phrase_five_clauses(true, 0, 10, 5)]
+    #[case::phrase_six_clauses(true, 0, 10, 6)]
+    fn test_bulk_and_matches_classic(
+        #[case] phrase: bool,
+        #[case] slop: u32,
+        #[case] limit: usize,
+        #[case] num_clauses: usize,
+    ) {
+        let num_docs = (BLOCK_SIZE * 8 + 37) as u32;
+        let mut docs = DocSet::default();
+        for doc_id in 0..num_docs {
+            docs.append(u64::from(doc_id), 32 + doc_id % 57);
+        }
+
+        // Clauses with different densities; membership comes from a cheap
+        // deterministic mix so docs scatter across blocks. Clause count picks
+        // the dedicated (1-3) or generic (4+) merge kernel.
+        let clause_docs = |modulus: u32, salt: u32| -> Vec<u32> {
+            (0..num_docs)
+                .filter(|doc| (doc.wrapping_mul(2654435761).wrapping_add(salt)) % modulus < 2)
+                .collect()
+        };
+        let clauses = [
+            clause_docs(3, 7),
+            clause_docs(4, 13),
+            clause_docs(5, 29),
+            clause_docs(3, 41),
+            clause_docs(3, 7),
+            clause_docs(4, 13),
+        ][..num_clauses]
+            .to_vec();
+
+        let build_postings = || {
+            clauses
+                .iter()
+                .enumerate()
+                .map(|(term_pos, doc_ids)| {
+                    let list = if phrase {
+                        // Roughly half of each clause's docs put the token at
+                        // position base+term_pos (forming the phrase); the
+                        // rest scatter so the position check has misses.
+                        let positions = doc_ids
+                            .iter()
+                            .map(|&doc| {
+                                if doc % 2 == 0 {
+                                    vec![5 + term_pos as u32, 40 + (doc % 3)]
+                                } else {
+                                    vec![20 + (term_pos as u32) * 4]
+                                }
+                            })
+                            .collect::<Vec<_>>();
+                        generate_posting_list_with_positions(doc_ids.clone(), positions, 8.0, true)
+                    } else {
+                        generate_posting_list(doc_ids.clone(), 8.0, None, true)
+                    };
+                    PostingIterator::with_query_weight(
+                        format!("t{term_pos}"),
+                        term_pos as u32,
+                        term_pos as u32,
+                        1.0 + term_pos as f32 * 0.5,
+                        list,
+                        docs.len(),
+                    )
+                })
+                .collect::<Vec<_>>()
+        };
+
+        let mut params = FtsSearchParams::default().with_limit(Some(limit));
+        if phrase {
+            params.phrase_slop = Some(slop);
+        }
+
+        let normalize = |result: Vec<DocCandidate>| {
+            let mut rows = result
+                .into_iter()
+                .map(|candidate| {
+                    (
+                        candidate.posting_doc_id,
+                        candidate.doc_length,
+                        candidate.freqs,
+                        match candidate.addr {
+                            CandidateAddr::RowId(row_id) => row_id,
+                            CandidateAddr::Pending(doc_id) => u64::from(doc_id),
+                        },
+                    )
+                })
+                .collect::<Vec<_>>();
+            rows.sort_unstable();
+            rows
+        };
+
+        let run = |mode| {
+            let mut wand = Wand::new(
+                Operator::And,
+                build_postings().into_iter(),
+                &docs,
+                UnitScorer,
+            )
+            .with_bulk_and_mode(mode);
+            let rows = normalize(
+                wand.search(
+                    &params,
+                    Arc::new(RowAddrMask::default()),
+                    &NoOpMetricsCollector,
+                )
+                .unwrap(),
+            );
+            let used_bulk = wand.bulk_and_searches > 0;
+            (rows, used_bulk)
+        };
+
+        let (bulk, bulk_used) = run(BulkAndMode::On);
+        let (classic, classic_used) = run(BulkAndMode::Off);
+        let (auto, auto_used) = run(BulkAndMode::Auto);
+        assert!(bulk_used, "on should use bulk conjunction search");
+        assert!(!classic_used, "off should use classic conjunction search");
+        assert_eq!(auto_used, matches!(num_clauses, 2 | 3));
+        assert!(!bulk.is_empty(), "test corpus should produce matches");
+        assert_eq!(bulk, classic);
+        assert_eq!(auto, classic);
     }
 }
diff --git a/rust/lance-index/src/scalar/json.rs b/rust/lance-index/src/scalar/json.rs
index 09678d6787e..ab37554a1f9 100644
--- a/rust/lance-index/src/scalar/json.rs
+++ b/rust/lance-index/src/scalar/json.rs
@@ -8,22 +8,24 @@ use std::{
 };
 
 use arrow_array::{Array, LargeBinaryArray, RecordBatch, StructArray, UInt8Array};
-use arrow_schema::{DataType, Field, Field as ArrowField, Schema};
+use arrow_schema::{DataType, Field, Field as ArrowField, Schema, SortOptions};
 use async_trait::async_trait;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::{
     execution::SendableRecordBatchStream,
-    physical_plan::{ExecutionPlan, projection::ProjectionExec},
+    physical_plan::{ExecutionPlan, projection::ProjectionExec, sorts::sort::SortExec},
 };
 use datafusion_common::{ScalarValue, config::ConfigOptions};
 use datafusion_expr::{Expr, Operator, ScalarUDF};
 use datafusion_physical_expr::{
-    PhysicalExpr, ScalarFunctionExpr,
+    PhysicalExpr, PhysicalSortExpr, ScalarFunctionExpr,
     expressions::{Column, Literal},
 };
 use futures::StreamExt;
 use lance_core::deepsize::DeepSizeOf;
-use lance_datafusion::exec::{LanceExecutionOptions, OneShotExec, get_session_context};
+use lance_datafusion::exec::{
+    LanceExecutionOptions, OneShotExec, execute_plan, get_session_context,
+};
 use lance_datafusion::udf::json::JsonbType;
 use prost::Message;
 use roaring::RoaringBitmap;
@@ -40,7 +42,8 @@ use crate::{
         UpdateCriteria,
         expression::{IndexedExpression, ScalarIndexExpr, ScalarIndexSearch, ScalarQueryParser},
         registry::{
-            BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingRequest, VALUE_COLUMN_NAME,
+            BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
+            VALUE_COLUMN_NAME,
         },
     },
 };
@@ -354,13 +357,29 @@ impl ScalarQueryParser for JsonQueryParser {
 pub struct JsonTrainingRequest {
     parameters: JsonIndexParameters,
     target_request: Box<dyn TrainingRequest>,
+    criteria: TrainingCriteria,
 }
 
 impl JsonTrainingRequest {
     pub fn new(parameters: JsonIndexParameters, target_request: Box<dyn TrainingRequest>) -> Self {
+        let target_criteria = target_request.criteria();
+        // The scanner can only sort its output by the raw JSON column, not by the value
+        // at `path` that this plugin extracts from it, so a `Values`-ordered scan here
+        // would sort by the wrong key and still need re-sorting after extraction. Ask
+        // for unordered input instead and let `train_index` sort the extracted value
+        // stream itself, once, right before handing it to the target trainer.
+        //
+        // This is safe for `Addresses` too: `scan_training_data` only special-cases
+        // `Values` (it calls `order_by` only then); an `Addresses` or `None` criteria
+        // both fall through to the same unordered-scan behavior, since the scan already
+        // returns rows in row-address order by default.
+        let mut criteria = TrainingCriteria::new(TrainingOrdering::None);
+        criteria.needs_row_ids = target_criteria.needs_row_ids;
+        criteria.needs_row_addrs = target_criteria.needs_row_addrs;
         Self {
             parameters,
             target_request,
+            criteria,
         }
     }
 }
@@ -371,7 +390,7 @@ impl TrainingRequest for JsonTrainingRequest {
     }
 
     fn criteria(&self) -> &TrainingCriteria {
-        self.target_request.criteria()
+        &self.criteria
     }
 }
 
@@ -665,6 +684,35 @@ impl JsonIndexPlugin {
             futures::stream::iter(converted_batches.into_iter().map(Ok)),
         )))
     }
+
+    /// Sort a `(value, row_id)` stream ascending by value.
+    ///
+    /// Target index types that require `TrainingOrdering::Values` (e.g. btree, whose
+    /// per-page min/max stats are taken from the first/last row of each page) need this
+    /// as the only sort in the JSON training path: `JsonTrainingRequest` requests
+    /// unordered input from the scanner, since the scanner can only sort on the raw
+    /// JSON column, not on the value at `path`.
+    async fn sort_stream_by_value(
+        data: SendableRecordBatchStream,
+    ) -> Result<SendableRecordBatchStream> {
+        let input = Arc::new(OneShotExec::new(data));
+        let value_idx = input.schema().index_of(VALUE_COLUMN_NAME)?;
+        let sort_expr = PhysicalSortExpr {
+            expr: Arc::new(Column::new(VALUE_COLUMN_NAME, value_idx)),
+            options: SortOptions {
+                descending: false,
+                nulls_first: true,
+            },
+        };
+        let plan = Arc::new(SortExec::new([sort_expr].into(), input));
+        execute_plan(
+            plan,
+            LanceExecutionOptions {
+                use_spilling: true,
+                ..Default::default()
+            },
+        )
+    }
 }
 
 #[async_trait]
@@ -720,6 +768,21 @@ impl BasicTrainer for JsonIndexPlugin {
         let converted_stream =
             Self::convert_stream_by_type(data_stream, inferred_type.clone()).await?;
 
+        // `JsonTrainingRequest::criteria()` asked the scanner for unordered input (see
+        // its constructor), since the scanner can only sort on the raw JSON column, not
+        // on the value at `path`. If the target index needs value-ordered input, this is
+        // the one place that sort happens: on the extracted value, after extraction.
+        //
+        // Deliberately `request.target_request.criteria()` here, not `request.criteria()`:
+        // the latter is `JsonTrainingRequest`'s own criteria, which is always `None` (that's
+        // what asked the scanner for unordered input above) and would never take this branch.
+        let converted_stream =
+            if request.target_request.criteria().ordering == TrainingOrdering::Values {
+                Self::sort_stream_by_value(converted_stream).await?
+            } else {
+                converted_stream
+            };
+
         // Update the target request with inferred type
         let registry = self.registry()?;
         let target_plugin = registry.get_plugin_by_name(&request.parameters.target_index_type)?;
@@ -837,8 +900,11 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::scalar::{SargableQuery, TextQuery};
     use arrow_array::{ArrayRef, RecordBatch};
     use arrow_schema::{DataType, Field, Schema};
+    use rstest::rstest;
+    use std::ops::Bound;
     use std::sync::Arc;
 
     // Note: The old test_detect_json_value_type test has been removed as we now use
@@ -978,4 +1044,280 @@ mod tests {
 
         assert_eq!(inferred_type, DataType::Utf8);
     }
+
+    /// Trains a JSON-path index of `target_index_type` over `json_docs` (fed to the
+    /// trainer in exactly the given order, with row ids `0..json_docs.len()`) and
+    /// returns the loaded index. `store` is a caller-owned `LanceIndexStore` so the
+    /// caller controls how long the backing `TempObjDir` stays alive.
+    async fn train_and_load_json_index(
+        store: Arc<dyn IndexStore>,
+        target_index_type: &str,
+        path: &str,
+        json_docs: &[&str],
+    ) -> Arc<dyn ScalarIndex> {
+        use crate::progress::noop_progress;
+        use arrow_array::{LargeBinaryArray, UInt64Array};
+        use futures::stream;
+
+        let jsonb: Vec<Vec<u8>> = json_docs
+            .iter()
+            .map(|s| s.parse::<jsonb::OwnedJsonb>().unwrap().to_vec())
+            .collect();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(VALUE_COLUMN_NAME, DataType::LargeBinary, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(LargeBinaryArray::from(
+                    jsonb.iter().map(|v| Some(v.as_slice())).collect::<Vec<_>>(),
+                )) as ArrayRef,
+                Arc::new(UInt64Array::from_iter_values(0..json_docs.len() as u64)) as ArrayRef,
+            ],
+        )
+        .unwrap();
+        let data = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            stream::iter(vec![Ok(batch)]),
+        )) as SendableRecordBatchStream;
+
+        let registry = IndexPluginRegistry::with_default_plugins();
+        let plugin = registry.get_plugin_by_name("json").unwrap();
+        let trainer = plugin.basic_trainer().unwrap();
+        let params = format!(r#"{{"target_index_type":"{target_index_type}","path":"{path}"}}"#);
+        let request = trainer
+            .new_training_request(
+                &params,
+                &Field::new(VALUE_COLUMN_NAME, DataType::LargeBinary, true),
+            )
+            .unwrap();
+
+        // The scanner must be asked for unordered input: only this plugin knows the
+        // order of the extracted value, so sorting on the raw JSON column would be
+        // wasted work that either goes unused (non-`Values` targets) or still leaves
+        // the extracted stream unsorted (`Values` targets, see below).
+        assert_eq!(request.criteria().ordering, TrainingOrdering::None);
+
+        let created = trainer
+            .train_index(data, store.as_ref(), request, None, noop_progress())
+            .await
+            .unwrap();
+
+        plugin
+            .load_index(store, &created.index_details, None, &LanceCache::no_cache())
+            .await
+            .unwrap()
+    }
+
+    fn local_json_index_store() -> (Arc<dyn IndexStore>, lance_core::utils::tempfile::TempObjDir) {
+        use crate::scalar::lance_format::LanceIndexStore;
+        use lance_core::utils::tempfile::TempObjDir;
+        use lance_io::object_store::ObjectStore;
+
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        )) as Arc<dyn IndexStore>;
+        (store, tmpdir)
+    }
+
+    /// Regression test for https://github.com/lance-format/lance/issues/7485.
+    ///
+    /// A JSON-path btree index over float values returned wrong results because the
+    /// btree trainer assumes its input arrives sorted by value (page min/max come from
+    /// the first/last row of each page), but the value at `path` is extracted by this
+    /// plugin *after* the scanner has already produced its rows, so a scan sorted on
+    /// the raw JSON column does not sort the extracted value. This exercises the fix
+    /// end to end: `JsonTrainingRequest::criteria()` must ask for unordered input (so
+    /// the scanner does not waste time sorting on the wrong key), and `train_index` must
+    /// sort the extracted value stream itself before training the target btree.
+    ///
+    /// Rows are fed in raw storage order (not sorted by value) to simulate what an
+    /// unordered scan would produce.
+    ///
+    /// Each case below runs a spilling `SortExec` that reserves a non-spillable merge
+    /// buffer from the process-wide cached DataFusion memory pool (see
+    /// `get_session_context`); running the cases concurrently contends for that shared
+    /// pool and can spuriously exhaust it, so this guard serializes them.
+    static FLOAT_INDEX_CASE_GUARD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+    #[rstest]
+    #[case::range_gt_zero(
+        SargableQuery::Range(Bound::Excluded(ScalarValue::Float64(Some(0.0))), Bound::Unbounded),
+        vec![0, 1]
+    )]
+    #[case::range_gte_page_min(
+        SargableQuery::Range(Bound::Included(ScalarValue::Float64(Some(10.5))), Bound::Unbounded),
+        vec![0, 1]
+    )]
+    #[case::equals_non_exact_float(
+        SargableQuery::Equals(ScalarValue::Float64(Some(40.1))),
+        vec![1]
+    )]
+    #[case::equals_exact_float(SargableQuery::Equals(ScalarValue::Float64(Some(10.5))), vec![0])]
+    #[case::range_covers_all(
+        SargableQuery::Range(Bound::Unbounded, Bound::Excluded(ScalarValue::Float64(Some(100.0)))),
+        vec![0, 1, 2]
+    )]
+    #[tokio::test]
+    async fn test_json_float_btree_index_unsorted_input(
+        #[case] query: SargableQuery,
+        #[case] expected: Vec<u64>,
+    ) {
+        let _guard = FLOAT_INDEX_CASE_GUARD.lock().await;
+        use crate::metrics::NoOpMetricsCollector;
+        use lance_select::RowAddrTreeMap;
+
+        // row0=10.5, row1=40.1, row2=-3.2: storage order does not match ascending value
+        // order (-3.2, 10.5, 40.1), so a btree trained on this order without an explicit
+        // value sort would record a corrupted page max of -3.2.
+        let (store, _tmpdir) = local_json_index_store();
+        let index = train_and_load_json_index(
+            store,
+            "btree",
+            "latitude",
+            &[
+                r#"{"latitude": 10.5}"#,
+                r#"{"latitude": 40.1}"#,
+                r#"{"latitude": -3.2}"#,
+            ],
+        )
+        .await;
+
+        let json_query = JsonQuery::new(Arc::new(query.clone()), "latitude".to_string());
+        let result = index
+            .search(&json_query, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert_eq!(
+            result,
+            SearchResult::exact(RowAddrTreeMap::from_iter(expected.iter().copied())),
+            "query {query:?}"
+        );
+    }
+
+    /// Regression test for a null value at `path` surviving `sort_stream_by_value`.
+    ///
+    /// `sort_stream_by_value` sorts the extracted `(value, row_id)` stream with
+    /// `nulls_first: true`. This checks that a null row's row_id stays paired with its
+    /// (null) value through that sort -- if the sort ever reordered values without their
+    /// row_ids, a null-valued row could be attributed to the wrong id -- and that the
+    /// resulting btree still answers `IsNull` and non-null range/equality queries
+    /// correctly with nulls mixed in and fed out of value order.
+    ///
+    /// Row 1's `path` is missing entirely, which is what actually produces a null in the
+    /// extracted value column (`extract_json_path_with_type` returns `None`, which
+    /// `json_extract_with_type_impl` turns into an arrow-null). An explicit JSON `null`
+    /// literal at `path` (e.g. `{"v": null}`) is a different, pre-existing case that
+    /// `convert_stream_by_type` does not yet handle (it tries to deserialize the JSONB
+    /// `null` bytes as the inferred type and errors) -- unrelated to this fix, so it's
+    /// out of scope here.
+    #[tokio::test]
+    async fn test_json_btree_index_null_at_path() {
+        use crate::metrics::NoOpMetricsCollector;
+        use lance_select::RowAddrTreeMap;
+
+        let _guard = FLOAT_INDEX_CASE_GUARD.lock().await;
+        let (store, _tmpdir) = local_json_index_store();
+        let index = train_and_load_json_index(
+            store,
+            "btree",
+            "v",
+            &[
+                r#"{"v": 40.1}"#,  // row 0
+                r#"{"other": 1}"#, // row 1: path missing -> null
+                r#"{"v": -3.2}"#,  // row 2
+                r#"{"v": 10.5}"#,  // row 3
+            ],
+        )
+        .await;
+
+        let search = |query: SargableQuery| {
+            let index = index.clone();
+            let json_query = JsonQuery::new(Arc::new(query), "v".to_string());
+            async move {
+                index
+                    .search(&json_query, &NoOpMetricsCollector)
+                    .await
+                    .unwrap()
+            }
+        };
+
+        // Range/equality queries carry row 1 in `nulls` (three-valued logic: `NULL > 0`
+        // is unknown, not false -- see `NullableRowAddrSet`), which is pre-existing
+        // btree/framework behavior. Asserting it exactly here is exactly the property
+        // this test targets: row 1's row_id must stay paired with its null value
+        // through `sort_stream_by_value`, not just be excluded from `selected`.
+        assert_eq!(
+            search(SargableQuery::IsNull()).await,
+            SearchResult::exact(RowAddrTreeMap::from_iter([1u64])),
+            "IsNull"
+        );
+        assert_eq!(
+            search(SargableQuery::Range(
+                Bound::Excluded(ScalarValue::Float64(Some(0.0))),
+                Bound::Unbounded,
+            ))
+            .await,
+            SearchResult::exact(RowAddrTreeMap::from_iter([0u64, 3]))
+                .with_nulls(RowAddrTreeMap::from_iter([1u64])),
+            "> 0"
+        );
+        assert_eq!(
+            search(SargableQuery::Equals(ScalarValue::Float64(Some(40.1)))).await,
+            SearchResult::exact(RowAddrTreeMap::from_iter([0u64]))
+                .with_nulls(RowAddrTreeMap::from_iter([1u64])),
+            "= 40.1"
+        );
+        assert_eq!(
+            search(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Excluded(ScalarValue::Float64(Some(100.0))),
+            ))
+            .await,
+            SearchResult::exact(RowAddrTreeMap::from_iter([0u64, 2, 3]))
+                .with_nulls(RowAddrTreeMap::from_iter([1u64])),
+            "< 100 (null is neither < 100 nor >= 100)"
+        );
+    }
+
+    /// Regression coverage for the non-`Values`-ordering branch in `train_index`: a
+    /// JSON-path index over a target that does not need value-ordered input (ngram
+    /// requires `TrainingOrdering::None`) must skip `sort_stream_by_value` entirely and
+    /// still produce correct results from rows fed out of value order.
+    #[tokio::test]
+    async fn test_json_ngram_index_skips_value_sort() {
+        use crate::metrics::NoOpMetricsCollector;
+        use lance_select::RowAddrTreeMap;
+
+        let (store, _tmpdir) = local_json_index_store();
+        let index = train_and_load_json_index(
+            store,
+            "ngram",
+            "tag",
+            &[
+                r#"{"tag": "unique-charlie"}"#,
+                r#"{"tag": "unique-alpha"}"#,
+                r#"{"tag": "unique-bravo"}"#,
+            ],
+        )
+        .await;
+
+        let json_query = JsonQuery::new(
+            Arc::new(TextQuery::StringContains("unique-bravo".to_string())),
+            "tag".to_string(),
+        );
+        let result = index
+            .search(&json_query, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert_eq!(
+            result,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([2u64])),
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/lance_format.rs b/rust/lance-index/src/scalar/lance_format.rs
index 0be5707d859..76ebd0cf0da 100644
--- a/rust/lance-index/src/scalar/lance_format.rs
+++ b/rust/lance-index/src/scalar/lance_format.rs
@@ -13,11 +13,8 @@ use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, cache::LanceCache};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
 use lance_encoding::version::LanceFileVersion;
-use lance_file::previous::{
-    reader::FileReader as PreviousFileReader,
-    writer::{FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider},
-};
-use lance_file::reader::{self as current_reader, FileReaderOptions, ReaderProjection};
+use lance_file::previous::reader::FileReader as PreviousFileReader;
+use lance_file::reader::{FileReader as CurrentFileReader, FileReaderOptions, ReaderProjection};
 use lance_file::writer as current_writer;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
@@ -130,34 +127,6 @@ impl LanceIndexStore {
     }
 }
 
-#[async_trait]
-impl<M: PreviousManifestProvider + Send + Sync> IndexWriter for PreviousFileWriter<M> {
-    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64> {
-        let offset = self.tell().await?;
-        self.write(&[batch]).await?;
-        Ok(offset as u64)
-    }
-
-    async fn finish(&mut self) -> Result<IndexFile> {
-        let summary = Self::finish(self).await?;
-        Ok(IndexFile {
-            path: String::new(),
-            size_bytes: summary.size_bytes,
-        })
-    }
-
-    async fn finish_with_metadata(
-        &mut self,
-        metadata: HashMap<String, String>,
-    ) -> Result<IndexFile> {
-        let summary = Self::finish_with_metadata(self, &metadata).await?;
-        Ok(IndexFile {
-            path: String::new(),
-            size_bytes: summary.size_bytes,
-        })
-    }
-}
-
 struct LanceIndexWriter {
     path: String,
     inner: current_writer::FileWriter,
@@ -198,10 +167,14 @@ impl IndexWriter for LanceIndexWriter {
     }
 }
 
+/// Newtype wrapper to allow implementing IndexReader for PreviousFileReader (a foreign type)
+struct PreviousIndexReader(PreviousFileReader);
+
 #[async_trait]
-impl IndexReader for PreviousFileReader {
+impl IndexReader for PreviousIndexReader {
     async fn read_record_batch(&self, offset: u64, _batch_size: u64) -> Result<RecordBatch> {
-        self.read_batch(offset as i32, ReadBatchParams::RangeFull, self.schema())
+        self.0
+            .read_batch(offset as i32, ReadBatchParams::RangeFull, self.0.schema())
             .await
     }
 
@@ -211,36 +184,39 @@ impl IndexReader for PreviousFileReader {
         projection: Option<&[&str]>,
     ) -> Result<RecordBatch> {
         let projection = match projection {
-            Some(projection) => self.schema().project(projection)?,
-            None => self.schema().clone(),
+            Some(projection) => self.0.schema().project(projection)?,
+            None => self.0.schema().clone(),
         };
-        self.read_range(range, &projection).await
+        self.0.read_range(range, &projection).await
     }
 
     async fn num_batches(&self, _batch_size: u64) -> u32 {
-        self.num_batches() as u32
+        self.0.num_batches() as u32
     }
 
     fn num_rows(&self) -> usize {
-        self.len()
+        self.0.len()
     }
 
     fn schema(&self) -> &lance_core::datatypes::Schema {
-        Self::schema(self)
+        PreviousFileReader::schema(&self.0)
     }
 }
 
+/// Newtype wrapper to allow implementing IndexReader for CurrentFileReader (a foreign type)
+struct CurrentIndexReader(CurrentFileReader);
+
 #[async_trait]
-impl IndexReader for current_reader::FileReader {
+impl IndexReader for CurrentIndexReader {
     async fn read_record_batch(&self, offset: u64, batch_size: u64) -> Result<RecordBatch> {
         let start = offset * batch_size;
         let end = start + batch_size;
-        let end = end.min(self.num_rows());
+        let end = end.min(self.0.num_rows());
         self.read_range(start as usize..end as usize, None).await
     }
 
     async fn read_global_buffer(&self, n: u32) -> Result<Bytes> {
-        Self::read_global_buffer(self, n).await
+        CurrentFileReader::read_global_buffer(&self.0, n).await
     }
 
     async fn read_range(
@@ -250,19 +226,20 @@ impl IndexReader for current_reader::FileReader {
     ) -> Result<RecordBatch> {
         if range.is_empty() {
             return Ok(RecordBatch::new_empty(Arc::new(
-                self.schema().as_ref().into(),
+                self.0.schema().as_ref().into(),
             )));
         }
         let projection = if let Some(projection) = projection {
             ReaderProjection::from_column_names(
-                self.metadata().version(),
-                self.schema(),
+                self.0.metadata().version(),
+                self.0.schema(),
                 projection,
             )?
         } else {
-            ReaderProjection::from_whole_schema(self.schema(), self.metadata().version())
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
         };
         let batches = self
+            .0
             .read_stream_projected(
                 ReadBatchParams::Range(range),
                 u32::MAX,
@@ -284,7 +261,7 @@ impl IndexReader for current_reader::FileReader {
     ) -> Result<RecordBatch> {
         let empty_batch = || {
             Ok(RecordBatch::new_empty(Arc::new(
-                self.schema().as_ref().into(),
+                self.0.schema().as_ref().into(),
             )))
         };
         if ranges.is_empty() {
@@ -292,12 +269,12 @@ impl IndexReader for current_reader::FileReader {
         }
         let projection = if let Some(projection) = projection {
             ReaderProjection::from_column_names(
-                self.metadata().version(),
-                self.schema(),
+                self.0.metadata().version(),
+                self.0.schema(),
                 projection,
             )?
         } else {
-            ReaderProjection::from_whole_schema(self.schema(), self.metadata().version())
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
         };
         // `DecodeBatchScheduler::schedule_ranges` requires sorted,
         // non-overlapping ranges; sort internally and permute the
@@ -311,6 +288,7 @@ impl IndexReader for current_reader::FileReader {
             .collect();
         let total_rows: u64 = sorted_ranges.iter().map(|r| r.end - r.start).sum();
         let batches = self
+            .0
             .read_stream_projected(
                 ReadBatchParams::Ranges(sorted_ranges),
                 (total_rows as u32).max(1),
@@ -363,47 +341,48 @@ impl IndexReader for current_reader::FileReader {
     ) -> Result<Pin<Box<dyn lance_io::stream::RecordBatchStream>>> {
         if range.is_empty() {
             return Ok(Box::pin(lance_io::stream::RecordBatchStreamAdapter::new(
-                Arc::new(self.schema().as_ref().into()),
+                Arc::new(self.0.schema().as_ref().into()),
                 futures::stream::empty(),
             )));
         }
         let projection = if let Some(projection) = projection {
             ReaderProjection::from_column_names(
-                self.metadata().version(),
-                self.schema(),
+                self.0.metadata().version(),
+                self.0.schema(),
                 projection,
             )?
         } else {
-            ReaderProjection::from_whole_schema(self.schema(), self.metadata().version())
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
         };
-        self.read_stream_projected(
-            ReadBatchParams::Range(range),
-            4096,
-            2,
-            projection,
-            FilterExpression::no_filter(),
-        )
-        .await
+        self.0
+            .read_stream_projected(
+                ReadBatchParams::Range(range),
+                4096,
+                2,
+                projection,
+                FilterExpression::no_filter(),
+            )
+            .await
     }
 
     // V2 format has removed the row group concept,
     // so here we assume each batch is with 4096 rows.
     async fn num_batches(&self, batch_size: u64) -> u32 {
-        Self::num_rows(self).div_ceil(batch_size) as u32
+        CurrentFileReader::num_rows(&self.0).div_ceil(batch_size) as u32
     }
 
     fn num_rows(&self) -> usize {
-        Self::num_rows(self) as usize
+        CurrentFileReader::num_rows(&self.0) as usize
     }
 
     fn schema(&self) -> &lance_core::datatypes::Schema {
-        Self::schema(self)
+        CurrentFileReader::schema(&self.0)
     }
 
     fn file_size_bytes(&self) -> Option<u64> {
         // The manifest records each index file's size and passes it to the reader
         // at open, so it's already in metadata here (no extra I/O).
-        Some(self.metadata().file_size())
+        Some(self.0.metadata().file_size())
     }
 }
 
@@ -464,7 +443,7 @@ impl IndexStore for LanceIndexStore {
             .scheduler
             .open_file_with_priority(&path, self.io_priority, &cached_size)
             .await?;
-        match current_reader::FileReader::try_open(
+        match CurrentFileReader::try_open(
             file_scheduler,
             None,
             Arc::<DecoderPlugins>::default(),
@@ -473,7 +452,7 @@ impl IndexStore for LanceIndexStore {
         )
         .await
         {
-            Ok(reader) => Ok(Arc::new(reader)),
+            Ok(reader) => Ok(Arc::new(CurrentIndexReader(reader))),
             Err(e) => {
                 // If the error is a version conflict we can try to read the file with v1 reader
                 if let Error::VersionConflict { .. } = e {
@@ -484,7 +463,7 @@ impl IndexStore for LanceIndexStore {
                         Some(&self.metadata_cache),
                     )
                     .await?;
-                    Ok(Arc::new(file_reader))
+                    Ok(Arc::new(PreviousIndexReader(file_reader)))
                 } else {
                     Err(e)
                 }
@@ -558,7 +537,14 @@ impl IndexStore for LanceIndexStore {
     }
 
     async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
-        list_index_files_with_sizes(&self.object_store, &self.index_dir).await
+        let files = list_index_files_with_sizes(&self.object_store, &self.index_dir).await?;
+        Ok(files
+            .into_iter()
+            .map(|f| IndexFile {
+                path: f.path,
+                size_bytes: f.size_bytes,
+            })
+            .collect())
     }
 }
 
diff --git a/rust/lance-index/src/scalar/registry.rs b/rust/lance-index/src/scalar/registry.rs
index 39dfff2c2c8..b65dd543100 100644
--- a/rust/lance-index/src/scalar/registry.rs
+++ b/rust/lance-index/src/scalar/registry.rs
@@ -4,7 +4,7 @@
 use std::borrow::Cow;
 use std::sync::Arc;
 
-use arrow_schema::Field;
+use arrow_schema::{DataType, Field};
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use futures::future::BoxFuture;
@@ -18,46 +18,11 @@ use crate::progress::IndexBuildProgress;
 use crate::registry::IndexPluginRegistry;
 use crate::scalar::RowIdRemapper;
 use crate::scalar::{CreatedIndex, IndexStore, ScalarIndex, expression::ScalarQueryParser};
+// Re-export training types that were previously defined here
+pub use crate::scalar::{TrainingCriteria, TrainingOrdering};
 
 pub const VALUE_COLUMN_NAME: &str = "value";
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TrainingOrdering {
-    /// The input will arrive sorted by the value column in ascending order
-    Values,
-    /// The input will arrive sorted by the address column in ascending order
-    Addresses,
-    /// The input will arrive in an arbitrary order
-    None,
-}
-
-#[derive(Debug, Clone)]
-pub struct TrainingCriteria {
-    pub ordering: TrainingOrdering,
-    pub needs_row_ids: bool,
-    pub needs_row_addrs: bool,
-}
-
-impl TrainingCriteria {
-    pub fn new(ordering: TrainingOrdering) -> Self {
-        Self {
-            ordering,
-            needs_row_ids: false,
-            needs_row_addrs: false,
-        }
-    }
-
-    pub fn with_row_id(mut self) -> Self {
-        self.needs_row_ids = true;
-        self
-    }
-
-    pub fn with_row_addr(mut self) -> Self {
-        self.needs_row_addrs = true;
-        self
-    }
-}
-
 /// A trait object for plugin-specific training parameters and data requirements.
 ///
 /// Returned by [`BasicTrainer::new_training_request`]. The caller uses
@@ -278,6 +243,55 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
         // Return an empty JSON object as the default implementation
         Ok(serde_json::json!({}))
     }
+
+    /// Optionally create a seed writer for the given column.
+    ///
+    /// A seed writer observes column values during data file writes, accumulates
+    /// compact statistics in memory, and serializes them as a global buffer
+    /// embedded in the data file footer. The buffer is later harvested during
+    /// index updates to skip a full column scan.
+    ///
+    /// All parameters needed to construct the writer must be derivable from
+    /// `index_details` — this method must not perform any I/O. Return `Ok(None)`
+    /// if this index type does not support seed writing.
+    async fn create_seed_writer(
+        &self,
+        _field_path: &str,
+        _data_type: &DataType,
+        _index_details: &prost_types::Any,
+    ) -> Result<Option<Box<dyn super::seed::IndexSeedWriter>>> {
+        Ok(None)
+    }
+
+    /// Returns true if this index type may have seed buffers embedded in data
+    /// files for the given index configuration.
+    ///
+    /// When false the caller can skip opening data files to look for seeds
+    /// entirely, avoiding I/O for index types or configurations that never
+    /// write seeds.
+    fn might_use_seeds(&self, _index_details: &prost_types::Any) -> bool {
+        false
+    }
+
+    /// Attempt to update `reference_index` using pre-harvested `seeds` instead
+    /// of re-scanning column data.
+    ///
+    /// Each [`FragmentSeed`](super::seed::FragmentSeed) carries the raw bytes
+    /// written by the corresponding [`IndexSeedWriter`](super::seed::IndexSeedWriter)
+    /// and the original `metadata_value` stored in the data file, which the plugin
+    /// can use for compatibility validation (e.g. confirming `rows_per_zone`).
+    ///
+    /// Return `Ok(Some(created))` if the seed-based update succeeded, or
+    /// `Ok(None)` to signal that the caller should fall back to a full column scan.
+    async fn update_from_seeds(
+        &self,
+        _seeds: Vec<super::seed::FragmentSeed>,
+        _reference_index: Arc<dyn ScalarIndex>,
+        _index_details: &prost_types::Any,
+        _dest_store: &dyn IndexStore,
+    ) -> Result<Option<CreatedIndex>> {
+        Ok(None)
+    }
 }
 
 /// A boxed, `Send` future performing the storage-level load of a scalar index
diff --git a/rust/lance-index/src/scalar/rtree.rs b/rust/lance-index/src/scalar/rtree.rs
index 0d6b54a02ca..a83f2250ac5 100644
--- a/rust/lance-index/src/scalar/rtree.rs
+++ b/rust/lance-index/src/scalar/rtree.rs
@@ -10,7 +10,8 @@ use crate::scalar::registry::{
 use crate::scalar::rtree::sort::Sorter;
 use crate::scalar::{
     AnyQuery, BuiltinIndexType, CreatedIndex, GeoQuery, IndexFile, IndexReader, IndexStore,
-    IndexWriter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, UpdateCriteria,
+    IndexWriter, OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult,
+    UpdateCriteria,
 };
 use crate::{Index, IndexType, pb};
 use arrow_array::UInt32Array;
@@ -54,6 +55,24 @@ const RTREE_INDEX_VERSION: u32 = 0;
 const RTREE_PAGES_NAME: &str = "page_data.lance";
 const RTREE_NULLS_NAME: &str = "nulls.lance";
 
+fn validate_page_size(page_size: u32) -> Result<()> {
+    if page_size < 2 {
+        return Err(Error::invalid_input(
+            "RTree page_size must be at least 2".to_string(),
+        ));
+    }
+    Ok(())
+}
+
+fn validate_stored_page_size(page_size: u32, num_items: usize) -> Result<()> {
+    if page_size == 0 || (page_size == 1 && num_items > 1) {
+        return Err(Error::invalid_input(format!(
+            "stored RTree page_size {page_size} cannot represent {num_items} items"
+        )));
+    }
+    Ok(())
+}
+
 static BBOX_FIELD: LazyLock<Arc<ArrowField>> = LazyLock::new(|| {
     let bbox_type = RectType::new(Dimension::XY, Default::default());
     Arc::new(bbox_type.to_field("bbox", false))
@@ -140,7 +159,9 @@ pub struct RTreeMetadata {
 impl RTreeMetadata {
     pub fn new(page_size: u32, num_pages: u64, num_items: usize, bbox: BoundingBox) -> Self {
         let page_offsets = Self::calculate_page_offsets(num_items, page_size);
-        debug_assert_eq!(page_offsets.len(), num_pages as usize);
+        if page_size >= 2 {
+            debug_assert_eq!(page_offsets.len(), num_pages as usize);
+        }
         Self {
             page_size,
             num_pages,
@@ -151,6 +172,9 @@ impl RTreeMetadata {
     }
 
     fn calculate_page_offsets(num_items: usize, page_size: u32) -> Vec<usize> {
+        if page_size < 2 {
+            return Vec::new();
+        }
         let mut page_offsets = vec![];
         let mut cur_level_items = num_items;
         let mut cur_offset = 0;
@@ -281,6 +305,7 @@ impl RTreeIndex {
     ) -> Result<Arc<Self>> {
         let pages_reader = store.open_index_file(RTREE_PAGES_NAME).await?;
         let metadata = RTreeMetadata::from(&pages_reader.schema().metadata);
+        validate_stored_page_size(metadata.page_size, metadata.num_items)?;
         let nulls_reader = store.open_index_file(RTREE_NULLS_NAME).await?;
 
         Ok(Arc::new(Self {
@@ -428,6 +453,155 @@ impl RTreeIndex {
     }
 }
 
+fn filter_keeps_nothing(filter: &Option<OldIndexDataFilter>) -> bool {
+    match filter {
+        Some(OldIndexDataFilter::Fragments { to_keep, .. }) => to_keep.is_empty(),
+        Some(OldIndexDataFilter::RowIds(valid)) => valid.is_empty(),
+        None => false,
+    }
+}
+
+fn filter_rtree_data(
+    data: SendableRecordBatchStream,
+    filter: OldIndexDataFilter,
+) -> SendableRecordBatchStream {
+    let schema = data.schema();
+    let filtered = data.map(move |batch_result| {
+        let batch = batch_result?;
+        let row_ids = batch
+            .column_by_name(ROW_ID)
+            .and_then(|column| column.as_any().downcast_ref::<UInt64Array>())
+            .ok_or_else(|| Error::internal("expected UInt64Array for RTree row ids"))?;
+        let mask = filter.filter_row_ids(row_ids);
+        Ok(arrow_select::filter::filter_record_batch(&batch, &mask)?)
+    });
+    Box::pin(RecordBatchStreamAdapter::new(schema, filtered))
+}
+
+fn remap_rtree_data(
+    data: SendableRecordBatchStream,
+    remapper: Arc<dyn RowIdRemapper>,
+) -> SendableRecordBatchStream {
+    let schema = data.schema();
+    let remapped = data.map(move |batch_result| {
+        let batch = batch_result?;
+        // The row ID is column 1 in BBOX_ROWID_SCHEMA.
+        Ok(remapper.remap_row_ids_record_batch(batch, 1)?)
+    });
+    Box::pin(RecordBatchStreamAdapter::new(schema, remapped))
+}
+
+/// Merge caller-selected RTree segments into one self-contained segment.
+///
+/// Each source may supply a filter for rows that are still live. The merged index recomputes its
+/// bounding box from the retained, remapped rows and preserves retained null row IDs.
+///
+/// # Examples
+///
+/// ```no_run
+/// use std::sync::Arc;
+///
+/// use lance_core::Result;
+/// use lance_index::scalar::OldIndexDataFilter;
+/// use lance_index::scalar::lance_format::LanceIndexStore;
+/// use lance_index::scalar::rtree::{RTreeIndex, merge_rtree_indices};
+///
+/// async fn merge(
+///     segments: &[Arc<RTreeIndex>],
+///     destination: &LanceIndexStore,
+///     filters: &[Option<OldIndexDataFilter>],
+/// ) -> Result<()> {
+///     merge_rtree_indices(segments, destination, filters).await?;
+///     Ok(())
+/// }
+/// ```
+pub async fn merge_rtree_indices(
+    source_indices: &[Arc<RTreeIndex>],
+    dest_store: &dyn IndexStore,
+    old_data_filters: &[Option<OldIndexDataFilter>],
+) -> Result<CreatedIndex> {
+    if source_indices.is_empty() {
+        return Err(Error::invalid_input(
+            "merge_rtree_indices requires at least one source index",
+        ));
+    }
+    if source_indices.len() != old_data_filters.len() {
+        return Err(Error::invalid_input(format!(
+            "merge_rtree_indices received {} source indices but {} filters",
+            source_indices.len(),
+            old_data_filters.len()
+        )));
+    }
+
+    let first_contributing = source_indices
+        .iter()
+        .zip(old_data_filters)
+        .find(|(source, filter)| source.metadata.num_items > 0 && !filter_keeps_nothing(filter))
+        .or_else(|| {
+            source_indices
+                .iter()
+                .zip(old_data_filters)
+                .find(|(_, filter)| !filter_keeps_nothing(filter))
+        })
+        .map(|(source, _)| source)
+        .unwrap_or(&source_indices[0]);
+    let page_size = first_contributing.metadata.page_size;
+    validate_page_size(page_size)?;
+    let mut data_streams = Vec::with_capacity(source_indices.len());
+    let mut null_map = RowAddrTreeMap::new();
+
+    for (source, filter) in source_indices.iter().zip(old_data_filters) {
+        if filter_keeps_nothing(filter) {
+            continue;
+        }
+        if source.metadata.num_items > 0 && source.metadata.page_size != page_size {
+            return Err(Error::invalid_input(format!(
+                "cannot merge RTree segments with different page sizes: {} and {}",
+                page_size, source.metadata.page_size
+            )));
+        }
+        let mut source_nulls = source.search_null(&NoOpMetricsCollector).await?;
+        if let Some(remapper) = &source.frag_reuse_index {
+            source_nulls = remapper.remap_row_addrs_tree_map(&source_nulls);
+        }
+        if let Some(filter) = filter {
+            filter.retain_old_rows(&mut source_nulls);
+        }
+        null_map |= &source_nulls;
+
+        let mut data = source.as_ref().clone().into_data_stream().await?;
+        if let Some(remapper) = source.frag_reuse_index.clone() {
+            data = remap_rtree_data(data, remapper);
+        }
+        data_streams.push(match filter {
+            Some(filter) => filter_rtree_data(data, filter.clone()),
+            None => data,
+        });
+    }
+
+    let combined = Box::pin(RecordBatchStreamAdapter::new(
+        BBOX_ROWID_SCHEMA.clone(),
+        stream::select_all(data_streams),
+    ));
+    let tmpdir = Arc::new(TempDir::default());
+    let spill_store = Arc::new(LanceIndexStore::new(
+        Arc::new(ObjectStore::local()),
+        tmpdir.obj_path(),
+        Arc::new(LanceCache::no_cache()),
+    ));
+    let (bbox_data, mut stats) =
+        RTreeIndexPlugin::process_and_analyze_bbox_stream(combined, page_size, spill_store).await?;
+    stats.null_map = null_map;
+    let files =
+        RTreeIndexPlugin::train_rtree_index(bbox_data, stats, page_size, dest_store).await?;
+
+    Ok(CreatedIndex {
+        index_details: prost_types::Any::from_msg(&pb::RTreeIndexDetails::default())?,
+        index_version: RTREE_INDEX_VERSION,
+        files,
+    })
+}
+
 impl DeepSizeOf for RTreeIndex {
     fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let mut total_size = 0;
@@ -824,6 +998,7 @@ impl RTreeIndexPlugin {
         store: &dyn IndexStore,
         page_size: u32,
     ) -> Result<IndexFile> {
+        validate_page_size(page_size)?;
         let mut page_idx: u64 = 0;
         let mut writer = store
             .new_index_file(RTREE_PAGES_NAME, RTREE_PAGE_SCHEMA.clone())
@@ -915,6 +1090,9 @@ impl BasicTrainer for RTreeIndexPlugin {
         _field: &ArrowField,
     ) -> Result<Box<dyn TrainingRequest>> {
         let params = serde_json::from_str::<RTreeParameters>(params)?;
+        if let Some(page_size) = params.page_size {
+            validate_page_size(page_size)?;
+        }
         Ok(Box::new(RTreeTrainingRequest::new(params)))
     }
 
@@ -923,15 +1101,9 @@ impl BasicTrainer for RTreeIndexPlugin {
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         request: Box<dyn TrainingRequest>,
-        fragment_ids: Option<Vec<u32>>,
+        _fragment_ids: Option<Vec<u32>>,
         _progress: Arc<dyn crate::progress::IndexBuildProgress>,
     ) -> Result<CreatedIndex> {
-        if fragment_ids.is_some() {
-            return Err(Error::invalid_input_source(
-                "RTree index does not support fragment training".into(),
-            ));
-        }
-
         Self::validate_schema(&data.schema())?;
 
         let request = request
@@ -942,6 +1114,7 @@ impl BasicTrainer for RTreeIndexPlugin {
             .parameters
             .page_size
             .unwrap_or(DEFAULT_RTREE_PAGE_SIZE);
+        validate_page_size(page_size)?;
 
         let bbox_data = Self::convert_bbox_stream(data)?;
         let tmpdir = Arc::new(TempDir::default());
@@ -1012,6 +1185,7 @@ struct EncodedBatch {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle};
     use crate::metrics::NoOpMetricsCollector;
     use crate::scalar::registry::VALUE_COLUMN_NAME;
     use arrow_array::ArrayRef;
@@ -1026,6 +1200,25 @@ mod tests {
         RTreeMetadata::calculate_page_offsets(num_items, page_size).len() as u64
     }
 
+    #[test]
+    fn test_rejects_page_size_that_cannot_reduce_tree_levels() {
+        let plugin = RTreeIndexPlugin;
+        let field = ArrowField::new("geometry", DataType::Null, true);
+        let error = plugin
+            .new_training_request(r#"{"page_size":1}"#, &field)
+            .err()
+            .unwrap();
+        assert!(error.to_string().contains("page_size must be at least 2"));
+    }
+
+    #[test]
+    fn test_stored_page_size_preserves_single_item_compatibility() {
+        assert!(validate_stored_page_size(1, 1).is_ok());
+        assert!(validate_stored_page_size(1, 0).is_ok());
+        assert!(validate_stored_page_size(1, 2).is_err());
+        assert!(validate_stored_page_size(0, 0).is_err());
+    }
+
     fn convert_bbox_rowid_batch_stream(
         geo_array: &dyn GeoArrowArray,
         row_id_array: ArrayRef,
@@ -1178,6 +1371,103 @@ mod tests {
         assert_eq!(actual_nulls, expected_nulls);
     }
 
+    #[tokio::test]
+    async fn test_merge_rtree_indices_filters_rows_and_nulls() {
+        let point_type = PointType::new(Dimension::XY, Default::default());
+        let mut first_builder = PointBuilder::new(point_type.clone());
+        first_builder.push_point(Some(&geo_types::point!(x: 10.0, y: 10.0)));
+        first_builder.push_null();
+        let first = first_builder.finish();
+
+        let mut second_builder = PointBuilder::new(point_type);
+        second_builder.push_point(Some(&geo_types::point!(x: 100.0, y: 100.0)));
+        second_builder.push_null();
+        let second = second_builder.finish();
+        let empty = PointBuilder::new(PointType::new(Dimension::XY, Default::default())).finish();
+
+        let (empty_index, _empty_store, _empty_tmpdir) = train_index(&empty, Some(8)).await;
+        let (first_index, _first_store, _first_tmpdir) = train_index(&first, Some(4)).await;
+        let (second_index, _second_store, _second_tmpdir) = train_index(&second, Some(4)).await;
+        let mut near_origin = BoundingBox::new();
+        near_origin.add_rect(&Rect::new(
+            coord! { x: 9.0, y: 9.0 },
+            coord! { x: 11.0, y: 11.0 },
+        ));
+        let mut expected_geometry = RowAddrTreeMap::new();
+        expected_geometry.insert(0);
+        assert_eq!(
+            first_index
+                .search_bbox(near_origin, &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            expected_geometry
+        );
+        let merged_tmpdir = TempObjDir::default();
+        let merged_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            merged_tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let remapped_geometry = RowAddress::new_from_parts(2, 0).into();
+        let remapped_null = RowAddress::new_from_parts(2, 1).into();
+        expected_geometry = RowAddrTreeMap::new();
+        expected_geometry.insert(remapped_geometry);
+        let remapper = FragReuseIndexHandle(Arc::new(FragReuseIndex::new(
+            uuid::Uuid::new_v4(),
+            vec![HashMap::from([
+                (0, Some(remapped_geometry)),
+                (1, Some(remapped_null)),
+            ])],
+            FragReuseIndexDetails { versions: vec![] },
+        )));
+        let mut first_index = first_index.as_ref().clone();
+        first_index.frag_reuse_index = Some(Arc::new(remapper));
+
+        let mut keep_first_rows = RowAddrTreeMap::new();
+        keep_first_rows.insert(remapped_geometry);
+        keep_first_rows.insert(remapped_null);
+        merge_rtree_indices(
+            &[empty_index, Arc::new(first_index), second_index],
+            merged_store.as_ref(),
+            &[
+                None,
+                Some(OldIndexDataFilter::RowIds(keep_first_rows)),
+                Some(OldIndexDataFilter::RowIds(RowAddrTreeMap::new())),
+            ],
+        )
+        .await
+        .unwrap();
+
+        let merged = RTreeIndex::load(merged_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(merged.metadata.num_items, 1);
+        assert_eq!(merged.metadata.bbox.minx(), 10.0);
+        assert_eq!(merged.metadata.bbox.miny(), 10.0);
+        assert_eq!(merged.metadata.bbox.maxx(), 10.0);
+        assert_eq!(merged.metadata.bbox.maxy(), 10.0);
+        assert!(
+            merged.metadata.bbox.rect_intersects(&near_origin),
+            "merged bounds {:?} do not intersect {:?}",
+            merged.metadata.bbox,
+            near_origin
+        );
+        assert_eq!(
+            merged
+                .search_bbox(near_origin, &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            expected_geometry
+        );
+        let mut expected_null = RowAddrTreeMap::new();
+        expected_null.insert(remapped_null);
+        assert_eq!(
+            merged.search_null(&NoOpMetricsCollector).await.unwrap(),
+            expected_null
+        );
+    }
+
     #[tokio::test]
     async fn test_update_and_search() {
         fn gen_data(num_items: u32, frag_id: u32, nulls_addrs: &mut RowAddrTreeMap) -> RectArray {
diff --git a/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs b/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
index e6c10a20575..9816bfebd51 100644
--- a/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
+++ b/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use crate::Result;
 use crate::scalar::rtree::sort::Sorter;
 use arrow_array::{ArrayRef, UInt32Array};
 use arrow_schema::{ArrowError, DataType as ArrowDataType, Field as ArrowField, Field};
@@ -19,9 +18,9 @@ use datafusion_physical_expr::expressions::Column as DFColumn;
 use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};
 use geoarrow_array::array::from_arrow_array;
 use geoarrow_array::{GeoArrowArray, GeoArrowArrayAccessor};
+use lance_core::Result;
 use lance_datafusion::exec::{LanceExecutionOptions, OneShotExec, execute_plan};
 use lance_geo::bbox::{BoundingBox, bounding_box};
-use std::any::Any;
 use std::sync::Arc;
 
 const HILBERT_FIELD_NAME: &str = "_hilbert";
@@ -149,10 +148,6 @@ impl HilbertUDF {
 }
 
 impl ScalarUDFImpl for HilbertUDF {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn name(&self) -> &str {
         HILBERT_UDF_NAME
     }
diff --git a/rust/lance-index/src/scalar/seed.rs b/rust/lance-index/src/scalar/seed.rs
new file mode 100644
index 00000000000..eaa5766fafa
--- /dev/null
+++ b/rust/lance-index/src/scalar/seed.rs
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Index seed writers — compact per-fragment summaries embedded in data files.
+//!
+//! A seed writer observes column values as they are written to a data file,
+//! accumulates compact statistics in memory, and serializes them to a byte
+//! buffer that is embedded in the data file footer as a global buffer.
+//!
+//! The buffer can later be read back during index updates to reconstruct index
+//! statistics without re-scanning the column data.
+
+use arrow_array::ArrayRef;
+use bytes::Bytes;
+use lance_core::Result;
+
+/// Schema metadata key prefix for all seed buffers: `"lance.seed.<column_name>"`.
+pub const SEED_META_KEY_PREFIX: &str = "lance.seed.";
+
+/// A hook registered during data file writes that observes column values batch
+/// by batch, accumulates compact statistics in memory, and serializes them to
+/// a byte buffer that is embedded in the data file footer as a global buffer.
+///
+/// The buffer can later be read back during index updates to reconstruct index
+/// statistics without re-scanning the column data.
+pub trait IndexSeedWriter: Send + std::fmt::Debug {
+    /// The column this writer is interested in.
+    fn column_name(&self) -> &str;
+
+    /// Observe a slice of column values as they are written to the current fragment.
+    /// Called once per batch.
+    fn observe_batch(&mut self, values: &ArrayRef) -> Result<()>;
+
+    /// Serialize accumulated state to bytes and reset for the next fragment.
+    /// Returns `None` if no data was observed (empty fragment).
+    fn finish(&mut self) -> Result<Option<Bytes>>;
+
+    /// Schema metadata key used to record that a seed buffer was written.
+    /// Convention: `"lance.seed.<column_name>"`.
+    fn schema_metadata_key(&self) -> String;
+
+    /// Create a string to store in the file's schema metadata. This will normally
+    /// contain the buffer index (provided by the caller after `add_global_buffer`)
+    /// as well as any other information needed to validate or understand the seed
+    /// (e.g. `rows_per_zone` for zone map seeds).
+    fn schema_metadata_value(&self, buf_index: u32) -> String;
+}
+
+/// A pre-harvested seed buffer from a single fragment's data file.
+#[derive(Debug, Clone)]
+pub struct FragmentSeed {
+    pub fragment_id: u64,
+    pub bytes: Bytes,
+    /// The raw value that was stored in the data file's schema metadata under
+    /// the seed key (i.e. the output of [`IndexSeedWriter::schema_metadata_value`]).
+    /// Plugins can inspect this to validate that the seed is compatible with the
+    /// current index configuration before consuming `bytes`.
+    pub metadata_value: String,
+}
diff --git a/rust/lance-index/src/scalar/zoned.rs b/rust/lance-index/src/scalar/zoned.rs
index 7ceed851bae..9c090f5e1d8 100644
--- a/rust/lance-index/src/scalar/zoned.rs
+++ b/rust/lance-index/src/scalar/zoned.rs
@@ -92,13 +92,14 @@ where
     pub async fn train(
         mut self,
         stream: SendableRecordBatchStream,
-    ) -> Result<Vec<P::ZoneStatistics>> {
+    ) -> Result<(Vec<P::ZoneStatistics>, RowAddrTreeMap)> {
         let zone_size = usize::try_from(self.zone_capacity).map_err(|_| {
             Error::invalid_input("zone capacity does not fit into usize on this platform")
         })?;
 
         let mut batches = chunk_concat_stream(stream, zone_size);
         let mut zones = Vec::new();
+        let mut null_rows = RowAddrTreeMap::new();
         let mut current_fragment_id: Option<u64> = None;
         let mut current_zone_len: usize = 0;
         let mut zone_start_offset: Option<u64> = None;
@@ -155,6 +156,16 @@ where
                 self.processor
                     .process_chunk(&values.slice(batch_offset, take))?;
 
+                // Record exact row addresses for null values in this chunk.
+                let chunk = values.slice(batch_offset, take);
+                if chunk.null_count() > 0 {
+                    for i in 0..take {
+                        if values.is_null(batch_offset + i) {
+                            null_rows.insert(row_addr_col.value(batch_offset + i));
+                        }
+                    }
+                }
+
                 // Track the first and last row offsets to handle non-contiguous offsets
                 // after deletions. Zone length (offset span) is computed as (last - first + 1),
                 // not the actual row count.
@@ -200,7 +211,7 @@ where
             }
         }
 
-        Ok(zones)
+        Ok((zones, null_rows))
     }
 
     /// Flushes a non-empty zone and resets the processor state.
@@ -266,21 +277,21 @@ where
 }
 
 /// Helper that retrains zones from `stream` and appends them to the existing
-/// statistics. Useful for index update paths that need to merge new fragments
-/// into an existing zone list.
+/// statistics. Returns the combined zone list and the null-row bitmap for the
+/// new data only — callers are responsible for merging with any existing bitmap.
 pub async fn rebuild_zones<P>(
     existing: &[P::ZoneStatistics],
     trainer: ZoneTrainer<P>,
     stream: SendableRecordBatchStream,
-) -> Result<Vec<P::ZoneStatistics>>
+) -> Result<(Vec<P::ZoneStatistics>, RowAddrTreeMap)>
 where
     P: ZoneProcessor,
     P::ZoneStatistics: Clone,
 {
     let mut combined = existing.to_vec();
-    let mut new_zones = trainer.train(stream).await?;
+    let (mut new_zones, null_rows) = trainer.train(stream).await?;
     combined.append(&mut new_zones);
-    Ok(combined)
+    Ok((combined, null_rows))
 }
 
 #[cfg(test)]
@@ -362,7 +373,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones: offsets [0..=3], [4..=7], [8..=9]
         assert_eq!(stats.len(), 3);
@@ -393,7 +404,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Two zones, one per fragment (capacity=10 is large enough)
         assert_eq!(stats.len(), 2);
@@ -447,7 +458,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone containing the 3 valid rows (empty batches skipped)
         assert_eq!(stats.len(), 1);
@@ -469,7 +480,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 1).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones, one per row (capacity=1)
         assert_eq!(stats.len(), 3);
@@ -494,7 +505,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10000).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone containing all 100 rows (capacity is large enough)
         assert_eq!(stats.len(), 1);
@@ -530,7 +541,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Two zones: first 4 rows, then remaining 2 rows
         assert_eq!(stats.len(), 2);
@@ -561,7 +572,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 3).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones: frag 0 full zone, frag 0 partial (flushed at boundary), frag 1
         assert_eq!(stats.len(), 3);
@@ -602,7 +613,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Should create 2 zones (capacity=4):
         // Zone 0: rows at offsets [0, 1, 5, 7] (4 rows)
@@ -637,7 +648,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone with 3 rows, but offset span [0..=200] so length=201 due to large gaps
         assert_eq!(stats.len(), 1);
@@ -663,7 +674,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Should create 3 zones (one per fragment)
         assert_eq!(stats.len(), 3);
@@ -810,7 +821,7 @@ mod tests {
         ));
 
         let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap();
-        let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap();
+        let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap();
         // Existing zone should remain unchanged and new stats appended afterwards
         assert_eq!(rebuilt.len(), 2);
         assert_eq!(rebuilt[0].sum, 50);
@@ -840,7 +851,7 @@ mod tests {
         ));
 
         let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap();
-        let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap();
+        let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap();
         // Existing zone plus two new fragments should yield three total zones
         assert_eq!(rebuilt.len(), 3);
         assert_eq!(rebuilt[0].bound.fragment_id, 0);
diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs
index 770188f3378..b4a1d71261e 100644
--- a/rust/lance-index/src/scalar/zonemap.rs
+++ b/rust/lance-index/src/scalar/zonemap.rs
@@ -12,12 +12,12 @@
 //! false positives that require rechecking.
 //!
 //!
-use crate::Any;
 use crate::pbold;
 use crate::scalar::expression::{SargableQueryParser, ScalarQueryParser};
 use crate::scalar::registry::{
     BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
 };
+use crate::scalar::seed::IndexSeedWriter;
 use crate::scalar::{
     BuiltinIndexType, CreatedIndex, IndexFile, SargableQuery, ScalarIndexParams, UpdateCriteria,
     compute_next_prefix,
@@ -26,6 +26,7 @@ use lance_arrow_stats::StatisticsAccumulator;
 use lance_core::cache::{LanceCache, WeakLanceCache};
 use lance_core::utils::row_addr_remap::RowAddrRemap;
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::sync::LazyLock;
 
 use arrow_array::{
@@ -34,7 +35,8 @@ use arrow_array::{
 use arrow_schema::{DataType, Field};
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion_common::ScalarValue;
-use std::sync::Arc;
+use lance_select::RowAddrTreeMap;
+use std::{collections::HashMap, sync::Arc};
 
 use super::{AnyQuery, IndexStore, MetricsCollector, ScalarIndex, SearchResult};
 use crate::scalar::RowIdRemapper;
@@ -50,11 +52,12 @@ const ROWS_PER_ZONE_DEFAULT: u64 = 8192; // 1 zone every two batches
 
 const ZONEMAP_FILENAME: &str = "zonemap.lance";
 const ZONEMAP_SIZE_META_KEY: &str = "rows_per_zone";
+const NULL_BITMAP_META_KEY: &str = "null_bitmap";
 const ZONEMAP_INDEX_VERSION: u32 = 0;
 
 /// Basic stats about zonemap index
 #[derive(Debug, PartialEq, Clone)]
-struct ZoneMapStatistics {
+pub(crate) struct ZoneMapStatistics {
     min: ScalarValue,
     max: ScalarValue,
     null_count: u32,
@@ -107,9 +110,13 @@ pub struct ZoneMapIndex {
     data_type: DataType,
     // The maximum rows per zone provided by user
     rows_per_zone: u64,
+    use_seeds: bool,
     store: Arc<dyn IndexStore>,
     fri: Option<Arc<dyn RowIdRemapper>>,
     index_cache: WeakLanceCache,
+    // Exact set of null row addresses across all zones; None when loaded from an
+    // older index that did not persist this bitmap.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl std::fmt::Debug for ZoneMapIndex {
@@ -118,6 +125,7 @@ impl std::fmt::Debug for ZoneMapIndex {
             .field("zones", &self.zones)
             .field("data_type", &self.data_type)
             .field("rows_per_zone", &self.rows_per_zone)
+            .field("use_seeds", &self.use_seeds)
             .field("store", &self.store)
             .field("fri", &self.fri)
             .field("index_cache", &self.index_cache)
@@ -127,11 +135,16 @@ impl std::fmt::Debug for ZoneMapIndex {
 
 impl DeepSizeOf for ZoneMapIndex {
     fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
-        self.zones.deep_size_of_children(context)
+        self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context)
     }
 }
 
 impl ZoneMapIndex {
+    /// Returns the rows-per-zone parameter for this index.
+    pub fn rows_per_zone(&self) -> u64 {
+        self.rows_per_zone
+    }
+
     fn scalar_is_nan(value: &ScalarValue) -> bool {
         match value {
             ScalarValue::Float16(Some(value)) => value.is_nan(),
@@ -260,10 +273,8 @@ impl ZoneMapIndex {
                                     return Ok(zone.nan_count > 0);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(zone.nan_count > 0);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(zone.nan_count > 0);
                             }
                             _ => {}
                         }
@@ -290,10 +301,8 @@ impl ZoneMapIndex {
                                     return Ok(false); // Nothing is greater than NaN
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(false); // Nothing is greater than NaN
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(false); // Nothing is greater than NaN
                             }
                             _ => {}
                         }
@@ -317,10 +326,8 @@ impl ZoneMapIndex {
                                     return Ok(zone.nan_count > 0 || zone_min <= e);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(zone.nan_count > 0 || zone_min <= e);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(zone.nan_count > 0 || zone_min <= e);
                             }
                             _ => {}
                         }
@@ -340,10 +347,8 @@ impl ZoneMapIndex {
                                     return Ok(true);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(true);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(true);
                             }
                             _ => {}
                         }
@@ -454,6 +459,7 @@ impl ZoneMapIndex {
         store: Arc<dyn IndexStore>,
         fri: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
+        use_seeds: bool,
     ) -> Result<Arc<Self>>
     where
         Self: Sized,
@@ -469,12 +475,25 @@ impl ZoneMapIndex {
             .get(ZONEMAP_SIZE_META_KEY)
             .and_then(|bs| bs.parse().ok())
             .unwrap_or(ROWS_PER_ZONE_DEFAULT);
+
+        let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) {
+            let idx = idx_str.parse::<u32>().map_err(|e| {
+                Error::invalid_input(format!("invalid null bitmap buffer index: {e}"))
+            })?;
+            let bytes = index_file.read_global_buffer(idx).await?;
+            Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?)
+        } else {
+            None
+        };
+
         Ok(Arc::new(Self::try_from_serialized(
             zone_maps,
             store,
             fri,
             index_cache,
             rows_per_zone,
+            null_rows,
+            use_seeds,
         )?))
     }
 
@@ -484,6 +503,8 @@ impl ZoneMapIndex {
         fri: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
         rows_per_zone: u64,
+        null_rows: Option<RowAddrTreeMap>,
+        use_seeds: bool,
     ) -> Result<Self> {
         // The RecordBatch should have columns: min, max, null_count
         let min_col = data
@@ -542,9 +563,11 @@ impl ZoneMapIndex {
                 zones: Vec::new(),
                 data_type,
                 rows_per_zone,
+                use_seeds,
                 store,
                 fri,
                 index_cache: WeakLanceCache::from(index_cache),
+                null_rows,
             });
         }
 
@@ -573,9 +596,11 @@ impl ZoneMapIndex {
             zones,
             data_type,
             rows_per_zone,
+            use_seeds,
             store,
             fri,
             index_cache: WeakLanceCache::from(index_cache),
+            null_rows,
         })
     }
 }
@@ -626,11 +651,21 @@ impl ScalarIndex for ZoneMapIndex {
         metrics: &dyn MetricsCollector,
     ) -> Result<SearchResult> {
         let query = query.as_any().downcast_ref::<SargableQuery>().unwrap();
+        if let SargableQuery::IsNull() = query
+            && let Some(null_rows) = &self.null_rows
+        {
+            return Ok(SearchResult::exact(null_rows.clone()));
+        }
+
         search_zones(&self.zones, metrics, |zone| {
             self.evaluate_zone_against_query(zone, query)
         })
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        true
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
@@ -660,19 +695,29 @@ impl ScalarIndex for ZoneMapIndex {
         let options = ZoneMapIndexBuilderParams::new(self.rows_per_zone);
         let processor = ZoneMapProcessor::new(value_type.clone())?;
         let trainer = ZoneTrainer::new(processor, self.rows_per_zone)?;
-        let updated_zones = rebuild_zones(&self.zones, trainer, new_data).await?;
+        let (updated_zones, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?;
+
+        // Merge existing and new null rows.  If the existing index had no null bitmap
+        // (legacy format — null positions unknown), preserve that None: updating cannot
+        // recover the missing information, and claiming the result has zero nulls would
+        // be a false negative.  Only a full retrain produces a fresh, complete bitmap.
+        let merged_null_rows = self.null_rows.as_ref().map(|existing| {
+            let mut merged = existing.clone();
+            merged |= &new_null_rows;
+            merged
+        });
 
         // Serialize the combined zones back into the index file
         let mut builder = ZoneMapIndexBuilder::try_new(options, self.data_type.clone())?;
         builder.options.rows_per_zone = self.rows_per_zone;
         builder.maps = updated_zones;
-        let file = builder.write_index(dest_store).await?;
+        builder.null_rows = merged_null_rows;
+        let files = builder.write_index(dest_store).await?;
 
         Ok(CreatedIndex {
-            index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
-                .unwrap(),
+            index_details: make_zone_map_index_details(self.rows_per_zone, self.use_seeds),
             index_version: ZONEMAP_INDEX_VERSION,
-            files: vec![file],
+            files,
         })
     }
 
@@ -683,7 +728,10 @@ impl ScalarIndex for ZoneMapIndex {
     }
 
     fn derive_index_params(&self) -> Result<ScalarIndexParams> {
-        let params = serde_json::to_value(ZoneMapIndexBuilderParams::new(self.rows_per_zone))?;
+        let params = serde_json::to_value(ZoneMapIndexBuilderParams {
+            rows_per_zone: self.rows_per_zone,
+            use_seeds: Some(self.use_seeds),
+        })?;
         Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(&params))
     }
 
@@ -694,6 +742,38 @@ impl ScalarIndex for ZoneMapIndex {
     }
 }
 
+impl ZoneMapIndex {
+    async fn try_update_with_seeds(
+        &self,
+        seeds: &[crate::scalar::seed::FragmentSeed],
+        dest_store: &dyn IndexStore,
+    ) -> Result<Option<CreatedIndex>> {
+        let mut new_zones = self.zones.clone();
+        for seed in seeds {
+            let mut zones = ZoneMapSeedWriter::deserialize_seed(
+                seed.fragment_id,
+                &seed.bytes,
+                self.rows_per_zone,
+            )?;
+            new_zones.append(&mut zones);
+        }
+        new_zones.sort_by_key(|z| (z.bound.fragment_id, z.bound.start));
+
+        let mut builder = ZoneMapIndexBuilder::try_new(
+            ZoneMapIndexBuilderParams::new(self.rows_per_zone),
+            self.data_type.clone(),
+        )?;
+        builder.maps = new_zones;
+        let files = builder.write_index(dest_store).await?;
+
+        Ok(Some(CreatedIndex {
+            index_details: make_zone_map_index_details(self.rows_per_zone, self.use_seeds),
+            index_version: ZONEMAP_INDEX_VERSION,
+            files,
+        }))
+    }
+}
+
 /// Merge caller-selected ZoneMap segments into one self-contained segment.
 pub async fn merge_zonemap_indices(
     source_indices: &[&ZoneMapIndex],
@@ -704,9 +784,12 @@ pub async fn merge_zonemap_indices(
         Error::invalid_input("merge_zonemap_indices requires at least one source index")
     })?;
     let rows_per_zone = first.rows_per_zone;
+    let use_seeds = first.use_seeds;
     let data_type = first.data_type.clone();
 
     let mut zones = Vec::new();
+    let mut merged_null_rows = RowAddrTreeMap::new();
+    let mut any_missing_bitmap = false;
     for source in source_indices {
         if source.rows_per_zone != rows_per_zone {
             return Err(Error::invalid_input(format!(
@@ -730,18 +813,29 @@ pub async fn merge_zonemap_indices(
                 })
                 .cloned(),
         );
+        match &source.null_rows {
+            Some(null_rows) => {
+                let mut filtered = null_rows.clone();
+                filtered.retain_fragments(fragment_filter.iter());
+                merged_null_rows |= &filtered;
+            }
+            None => any_missing_bitmap = true,
+        }
     }
     zones.sort_by_key(|zone| (zone.bound.fragment_id, zone.bound.start));
 
     let mut builder =
         ZoneMapIndexBuilder::try_new(ZoneMapIndexBuilderParams::new(rows_per_zone), data_type)?;
     builder.maps = zones;
-    builder.write_index(dest_store).await?;
+    if !any_missing_bitmap {
+        builder.null_rows = Some(merged_null_rows);
+    }
+    let files = builder.write_index(dest_store).await?;
 
     Ok(CreatedIndex {
-        index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()).unwrap(),
+        index_details: make_zone_map_index_details(rows_per_zone, use_seeds),
         index_version: ZONEMAP_INDEX_VERSION,
-        files: dest_store.list_files_with_sizes().await?,
+        files,
     })
 }
 
@@ -753,6 +847,12 @@ fn default_rows_per_zone() -> u64 {
 pub struct ZoneMapIndexBuilderParams {
     #[serde(default = "default_rows_per_zone")]
     rows_per_zone: u64,
+    /// Whether to embed per-fragment seed buffers in data files for use during
+    /// incremental index updates. `None` means auto-detect based on column type
+    /// (see [`default_use_seeds`]). Resolved to a concrete `bool` during
+    /// training in [`ZoneMapIndexPlugin::new_training_request`].
+    #[serde(default)]
+    use_seeds: Option<bool>,
 }
 
 static DEFAULT_ROWS_PER_ZONE: LazyLock<u64> = LazyLock::new(|| {
@@ -766,13 +866,17 @@ impl Default for ZoneMapIndexBuilderParams {
     fn default() -> Self {
         Self {
             rows_per_zone: *DEFAULT_ROWS_PER_ZONE,
+            use_seeds: None,
         }
     }
 }
 
 impl ZoneMapIndexBuilderParams {
     pub fn new(rows_per_zone: u64) -> Self {
-        Self { rows_per_zone }
+        Self {
+            rows_per_zone,
+            use_seeds: None,
+        }
     }
 
     pub fn rows_per_zone(&self) -> u64 {
@@ -786,6 +890,10 @@ pub struct ZoneMapIndexBuilder {
 
     items_type: DataType,
     maps: Vec<ZoneMapStatistics>,
+    // None means "legacy index — null positions unknown"; Some means a complete bitmap.
+    // write_index omits the null-bitmap global buffer when this is None, preserving the
+    // legacy format so that downstream searches remain conservative.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl ZoneMapIndexBuilder {
@@ -794,6 +902,7 @@ impl ZoneMapIndexBuilder {
             options,
             items_type,
             maps: Vec::new(),
+            null_rows: None,
         })
     }
 
@@ -803,7 +912,9 @@ impl ZoneMapIndexBuilder {
     pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> {
         let processor = ZoneMapProcessor::new(self.items_type.clone())?;
         let trainer = ZoneTrainer::new(processor, self.options.rows_per_zone)?;
-        self.maps = trainer.train(batches_source).await?;
+        let (maps, null_rows) = trainer.train(batches_source).await?;
+        self.maps = maps;
+        self.null_rows = Some(null_rows);
         Ok(())
     }
 
@@ -856,7 +967,7 @@ impl ZoneMapIndexBuilder {
         Ok(RecordBatch::try_new(schema, columns)?)
     }
 
-    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<IndexFile> {
+    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
         let record_batch = self.zonemap_stats_as_batch()?;
 
         let mut file_schema = record_batch.schema().as_ref().clone();
@@ -869,12 +980,30 @@ impl ZoneMapIndexBuilder {
             .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema))
             .await?;
         index_file.write_record_batch(record_batch).await?;
-        index_file.finish().await
+
+        let zonemap_file = if let Some(null_rows) = self.null_rows {
+            let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size());
+            null_rows.serialize_into(&mut null_bitmap_bytes)?;
+            let null_bitmap_idx = index_file
+                .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes))
+                .await?;
+            index_file
+                .finish_with_metadata(HashMap::from([(
+                    NULL_BITMAP_META_KEY.to_string(),
+                    null_bitmap_idx.to_string(),
+                )]))
+                .await?
+        } else {
+            index_file.finish_with_metadata(HashMap::new()).await?
+        };
+
+        Ok(vec![zonemap_file])
     }
 }
 
 /// Index-specific processor that computes min/max statistics for each zone while the
 /// trainer takes care of chunking and fragment boundaries.
+#[derive(Debug)]
 struct ZoneMapProcessor {
     data_type: DataType,
     statistics: StatisticsAccumulator,
@@ -966,6 +1095,37 @@ impl ZoneProcessor for ZoneMapProcessor {
     }
 }
 
+fn make_zone_map_index_details(rows_per_zone: u64, use_seeds: bool) -> prost_types::Any {
+    prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails {
+        rows_per_zone: Some(rows_per_zone),
+        use_seeds: Some(use_seeds),
+    })
+    .unwrap()
+}
+
+/// Returns true when seed-based incremental updates should be enabled by
+/// default for the given column type.
+///
+/// Seeds pay off for variable-length types (strings, binary) — which can be
+/// arbitrarily wide — and fixed-width types wider than 8 bytes (e.g.
+/// Decimal128, FixedSizeBinary tensors). Fixed-width types ≤ 8 bytes (Int64,
+/// Float64, …) scan fast enough that the seed overhead is not worth it.
+fn default_use_seeds(data_type: &DataType) -> bool {
+    match data_type {
+        // Variable-length: width is unbounded, skipping scans is always valuable.
+        DataType::Utf8
+        | DataType::LargeUtf8
+        | DataType::Utf8View
+        | DataType::Binary
+        | DataType::LargeBinary
+        | DataType::BinaryView => true,
+        // Fixed-width types wider than 8 bytes.
+        DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => true,
+        DataType::FixedSizeBinary(n) => *n > 8,
+        _ => false,
+    }
+}
+
 #[derive(Debug, Default)]
 pub struct ZoneMapIndexPlugin;
 
@@ -974,8 +1134,7 @@ impl ZoneMapIndexPlugin {
         batches_source: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         options: Option<ZoneMapIndexBuilderParams>,
-    ) -> Result<IndexFile> {
-        // train_zonemap_index: calling scan_aligned_chunks
+    ) -> Result<Vec<IndexFile>> {
         let value_type = batches_source.schema().field(0).data_type().clone();
 
         let mut builder = ZoneMapIndexBuilder::try_new(options.unwrap_or_default(), value_type)?;
@@ -1022,7 +1181,11 @@ impl BasicTrainer for ZoneMapIndexPlugin {
             ));
         }
 
-        let params = serde_json::from_str::<ZoneMapIndexBuilderParams>(params)?;
+        let mut params = serde_json::from_str::<ZoneMapIndexBuilderParams>(params)?;
+        // Resolve None → type-based default so train_index always sees Some(bool).
+        if params.use_seeds.is_none() {
+            params.use_seeds = Some(default_use_seeds(field.data_type()));
+        }
 
         Ok(Box::new(ZoneMapIndexTrainingRequest::new(params)))
     }
@@ -1042,12 +1205,13 @@ impl BasicTrainer for ZoneMapIndexPlugin {
                     "must provide training request created by new_training_request".into(),
                 )
             })?;
-        let file = Self::train_zonemap_index(data, index_store, Some(request.params)).await?;
+        let rows_per_zone = request.params.rows_per_zone;
+        let use_seeds = request.params.use_seeds.unwrap_or(false);
+        let files = Self::train_zonemap_index(data, index_store, Some(request.params)).await?;
         Ok(CreatedIndex {
-            index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
-                .unwrap(),
+            index_details: make_zone_map_index_details(rows_per_zone, use_seeds),
             index_version: ZONEMAP_INDEX_VERSION,
-            files: vec![file],
+            files,
         })
     }
 }
@@ -1085,11 +1249,332 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin {
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
-        _index_details: &prost_types::Any,
+        index_details: &prost_types::Any,
         frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
-        Ok(ZoneMapIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
+        let use_seeds = index_details
+            .to_msg::<pbold::ZoneMapIndexDetails>()
+            .ok()
+            .and_then(|d| d.use_seeds)
+            .unwrap_or(false);
+        Ok(
+            ZoneMapIndex::load(index_store, frag_reuse_index, cache, use_seeds).await?
+                as Arc<dyn ScalarIndex>,
+        )
+    }
+
+    fn might_use_seeds(&self, index_details: &prost_types::Any) -> bool {
+        index_details
+            .to_msg::<pbold::ZoneMapIndexDetails>()
+            .ok()
+            .and_then(|d| d.use_seeds)
+            .unwrap_or(false)
+    }
+
+    async fn create_seed_writer(
+        &self,
+        field_path: &str,
+        data_type: &DataType,
+        index_details: &prost_types::Any,
+    ) -> Result<Option<Box<dyn crate::scalar::seed::IndexSeedWriter>>> {
+        if data_type.is_nested() {
+            return Ok(None);
+        }
+        let details = index_details.to_msg::<pbold::ZoneMapIndexDetails>().ok();
+        let Some(rows_per_zone) = details.as_ref().and_then(|d| d.rows_per_zone) else {
+            return Ok(None);
+        };
+        if !details.as_ref().and_then(|d| d.use_seeds).unwrap_or(false) {
+            return Ok(None);
+        }
+        Ok(Some(Box::new(ZoneMapSeedWriter::new(
+            field_path,
+            rows_per_zone,
+            data_type.clone(),
+        )?)))
+    }
+
+    async fn update_from_seeds(
+        &self,
+        seeds: Vec<crate::scalar::seed::FragmentSeed>,
+        reference_index: Arc<dyn ScalarIndex>,
+        index_details: &prost_types::Any,
+        dest_store: &dyn IndexStore,
+    ) -> Result<Option<CreatedIndex>> {
+        let Some(rows_per_zone) = index_details
+            .to_msg::<pbold::ZoneMapIndexDetails>()
+            .ok()
+            .and_then(|d| d.rows_per_zone)
+        else {
+            return Ok(None);
+        };
+
+        // Validate each seed was written with the same rows_per_zone.
+        for seed in &seeds {
+            let rpz_in_seed = seed
+                .metadata_value
+                .split_once(':')
+                .and_then(|(_, rpz)| rpz.parse::<u64>().ok());
+            if rpz_in_seed != Some(rows_per_zone) {
+                return Ok(None);
+            }
+        }
+
+        let Some(zone_map) = reference_index.as_any().downcast_ref::<ZoneMapIndex>() else {
+            return Ok(None);
+        };
+        zone_map.try_update_with_seeds(&seeds, dest_store).await
+    }
+}
+
+/// A seed writer that observes column values during data file writes and
+/// accumulates zone map statistics for later harvest during index updates.
+///
+/// Zone statistics are serialized as Arrow IPC bytes and embedded in the data
+/// file footer as a global buffer, keyed by `"lance.seed.<column_name>"`.
+#[derive(Debug)]
+pub struct ZoneMapSeedWriter {
+    column_name: String,
+    rows_per_zone: u64,
+    data_type: DataType,
+    completed_zones: Vec<ZoneMapStatistics>,
+    processor: ZoneMapProcessor,
+    rows_in_current_zone: u64,
+    next_zone_start: u64,
+}
+
+impl ZoneMapSeedWriter {
+    /// Create a new `ZoneMapSeedWriter` for the given column.
+    pub fn new(
+        column_name: impl Into<String>,
+        rows_per_zone: u64,
+        data_type: DataType,
+    ) -> Result<Self> {
+        if rows_per_zone == 0 {
+            return Err(lance_core::Error::invalid_input(
+                "rows_per_zone must be greater than zero",
+            ));
+        }
+        let processor = ZoneMapProcessor::new(data_type.clone())?;
+        Ok(Self {
+            column_name: column_name.into(),
+            rows_per_zone,
+            data_type,
+            completed_zones: Vec::new(),
+            processor,
+            rows_in_current_zone: 0,
+            next_zone_start: 0,
+        })
+    }
+
+    fn seed_batch_from_zones(
+        zones: &[ZoneMapStatistics],
+        data_type: &DataType,
+    ) -> Result<arrow_array::RecordBatch> {
+        let mins = if zones.is_empty() {
+            arrow_array::new_empty_array(data_type)
+        } else {
+            datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.min.clone()))?
+        };
+        let maxs = if zones.is_empty() {
+            arrow_array::new_empty_array(data_type)
+        } else {
+            datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.max.clone()))?
+        };
+        let null_counts =
+            arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.null_count));
+        let nan_counts =
+            arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.nan_count));
+        let zone_lengths =
+            arrow_array::UInt64Array::from_iter_values(zones.iter().map(|s| s.bound.length as u64));
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            Field::new("min", data_type.clone(), true),
+            Field::new("max", data_type.clone(), true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+
+        let columns: Vec<ArrayRef> = vec![
+            mins,
+            maxs,
+            Arc::new(null_counts) as ArrayRef,
+            Arc::new(nan_counts) as ArrayRef,
+            Arc::new(zone_lengths) as ArrayRef,
+        ];
+        Ok(arrow_array::RecordBatch::try_new(schema, columns)?)
+    }
+
+    /// Deserialize zone map seed bytes (Arrow IPC) into zone statistics.
+    ///
+    /// Returns a list of `ZoneMapStatistics` with bounds reconstructed from
+    /// the sequential layout (zone `i` starts at `i * rows_per_zone`).
+    pub(crate) fn deserialize_seed(
+        fragment_id: u64,
+        bytes: &bytes::Bytes,
+        rows_per_zone: u64,
+    ) -> Result<Vec<ZoneMapStatistics>> {
+        use arrow_ipc::reader::FileReader;
+        use std::io::Cursor;
+
+        let cursor = Cursor::new(bytes.as_ref());
+        let mut reader = FileReader::try_new(cursor, None).map_err(|e| {
+            lance_core::Error::invalid_input(format!("failed to read zone map seed IPC: {}", e))
+        })?;
+
+        let batch = match reader.next() {
+            Some(Ok(batch)) => batch,
+            Some(Err(e)) => {
+                return Err(lance_core::Error::invalid_input(format!(
+                    "failed to read zone map seed batch: {}",
+                    e
+                )));
+            }
+            None => return Ok(Vec::new()),
+        };
+
+        let min_col = batch
+            .column_by_name("min")
+            .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'min' column"))?;
+        let max_col = batch
+            .column_by_name("max")
+            .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'max' column"))?;
+        let null_count_col = batch
+            .column_by_name("null_count")
+            .ok_or_else(|| {
+                lance_core::Error::invalid_input("seed batch missing 'null_count' column")
+            })?
+            .as_any()
+            .downcast_ref::<arrow_array::UInt32Array>()
+            .ok_or_else(|| lance_core::Error::invalid_input("seed 'null_count' is not UInt32"))?;
+        let nan_count_col = batch
+            .column_by_name("nan_count")
+            .ok_or_else(|| {
+                lance_core::Error::invalid_input("seed batch missing 'nan_count' column")
+            })?
+            .as_any()
+            .downcast_ref::<arrow_array::UInt32Array>()
+            .ok_or_else(|| lance_core::Error::invalid_input("seed 'nan_count' is not UInt32"))?;
+        let zone_length_col = batch
+            .column_by_name("zone_length")
+            .ok_or_else(|| {
+                lance_core::Error::invalid_input("seed batch missing 'zone_length' column")
+            })?
+            .as_any()
+            .downcast_ref::<arrow_array::UInt64Array>()
+            .ok_or_else(|| lance_core::Error::invalid_input("seed 'zone_length' is not UInt64"))?;
+
+        let num_zones = batch.num_rows();
+        let mut zones = Vec::with_capacity(num_zones);
+        for i in 0..num_zones {
+            let zone_start = i as u64 * rows_per_zone;
+            let zone_length = zone_length_col.value(i) as usize;
+            zones.push(ZoneMapStatistics {
+                min: datafusion_common::ScalarValue::try_from_array(min_col, i)?,
+                max: datafusion_common::ScalarValue::try_from_array(max_col, i)?,
+                null_count: null_count_col.value(i),
+                nan_count: nan_count_col.value(i),
+                bound: ZoneBound {
+                    fragment_id,
+                    start: zone_start,
+                    length: zone_length,
+                },
+            });
+        }
+        Ok(zones)
+    }
+}
+
+impl IndexSeedWriter for ZoneMapSeedWriter {
+    fn column_name(&self) -> &str {
+        &self.column_name
+    }
+
+    fn observe_batch(&mut self, values: &ArrayRef) -> lance_core::Result<()> {
+        let mut offset = 0usize;
+
+        while offset < values.len() {
+            let remaining_in_zone = self.rows_per_zone - self.rows_in_current_zone;
+            let chunk_len = ((values.len() as u64 - offset as u64).min(remaining_in_zone)) as usize;
+            let chunk = values.slice(offset, chunk_len);
+            self.processor.process_chunk(&chunk)?;
+            self.rows_in_current_zone += chunk_len as u64;
+            offset += chunk_len;
+
+            if self.rows_in_current_zone >= self.rows_per_zone {
+                let bound = ZoneBound {
+                    fragment_id: 0,
+                    start: self.next_zone_start,
+                    length: self.rows_per_zone as usize,
+                };
+                let stats = self.processor.finish_zone(bound)?;
+                self.processor.reset()?;
+                self.completed_zones.push(stats);
+                self.next_zone_start += self.rows_per_zone;
+                self.rows_in_current_zone = 0;
+            }
+        }
+        Ok(())
+    }
+
+    fn finish(&mut self) -> lance_core::Result<Option<bytes::Bytes>> {
+        use arrow_ipc::writer::FileWriter;
+        use std::io::Cursor;
+
+        // Flush partial final zone
+        if self.rows_in_current_zone > 0 {
+            let bound = ZoneBound {
+                fragment_id: 0,
+                start: self.next_zone_start,
+                length: self.rows_in_current_zone as usize,
+            };
+            let stats = self.processor.finish_zone(bound)?;
+            self.processor.reset()?;
+            self.completed_zones.push(stats);
+            self.next_zone_start += self.rows_in_current_zone;
+            self.rows_in_current_zone = 0;
+        }
+
+        if self.completed_zones.is_empty() {
+            return Ok(None);
+        }
+
+        let batch = Self::seed_batch_from_zones(&self.completed_zones, &self.data_type)?;
+
+        // Serialize to Arrow IPC
+        let mut buf = Cursor::new(Vec::new());
+        {
+            let mut writer = FileWriter::try_new(&mut buf, batch.schema_ref()).map_err(|e| {
+                lance_core::Error::invalid_input(format!("failed to create IPC writer: {}", e))
+            })?;
+            writer.write(&batch).map_err(|e| {
+                lance_core::Error::invalid_input(format!("failed to write IPC batch: {}", e))
+            })?;
+            writer.finish().map_err(|e| {
+                lance_core::Error::invalid_input(format!("failed to finish IPC writer: {}", e))
+            })?;
+        }
+
+        // Reset state for next fragment
+        self.completed_zones.clear();
+        self.next_zone_start = 0;
+        self.processor = ZoneMapProcessor::new(self.data_type.clone())?;
+
+        Ok(Some(bytes::Bytes::from(buf.into_inner())))
+    }
+
+    fn schema_metadata_key(&self) -> String {
+        format!(
+            "{}{}",
+            crate::scalar::seed::SEED_META_KEY_PREFIX,
+            self.column_name
+        )
+    }
+
+    fn schema_metadata_value(&self, buf_index: u32) -> String {
+        format!("{}:{}", buf_index, self.rows_per_zone)
     }
 }
 
@@ -1117,13 +1602,14 @@ mod tests {
     use lance_datagen::ArrayGeneratorExt;
     use lance_datagen::{BatchCount, RowCount, array};
     use lance_io::object_store::ObjectStore;
-    use lance_select::{NullableRowAddrSet, RowAddrTreeMap};
+    use lance_select::RowAddrTreeMap;
 
     use crate::scalar::{
         SargableQuery, ScalarIndex, SearchResult,
         lance_format::LanceIndexStore,
         zonemap::{
             ZONEMAP_FILENAME, ZONEMAP_SIZE_META_KEY, ZoneMapIndex, ZoneMapIndexBuilderParams,
+            merge_zonemap_indices,
         },
     };
 
@@ -1197,7 +1683,7 @@ mod tests {
         .await
         .unwrap();
 
-        ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex")
     }
@@ -1302,7 +1788,7 @@ mod tests {
         log::debug!("Successfully wrote the index file");
 
         // Read the index file back and check its contents
-        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex");
         assert_eq!(index.zones.len(), 0);
@@ -1346,7 +1832,7 @@ mod tests {
         log::debug!("Successfully wrote the index file");
 
         // Read the index file back and check its contents
-        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex");
         assert_eq!(index.zones.len(), 10);
@@ -1398,9 +1884,10 @@ mod tests {
             .unwrap();
 
         // Verify the updated index has more zones
-        let updated_index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
-            .await
-            .expect("Failed to load updated ZoneMapIndex");
+        let updated_index =
+            ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
+                .await
+                .expect("Failed to load updated ZoneMapIndex");
 
         // Should have original 10 zones + 1 new zone (5000 rows with zone size 5000)
         assert_eq!(updated_index.zones.len(), 11);
@@ -1469,7 +1956,7 @@ mod tests {
             .unwrap();
 
         let cache = LanceCache::with_capacity(1024 * 1024);
-        let index = ZoneMapIndex::load(store.clone(), None, &cache)
+        let index = ZoneMapIndex::load(store.clone(), None, &cache, false)
             .await
             .unwrap();
 
@@ -1572,7 +2059,7 @@ mod tests {
         .unwrap();
 
         // Load the index
-        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex");
 
@@ -1676,7 +2163,7 @@ mod tests {
         // Test IsNull query (should match nothing since there are no null values)
         let query = SargableQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty()));
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
 
         // Test range queries with NaN bounds
         // Range with NaN as start bound (included)
@@ -1719,7 +2206,7 @@ mod tests {
         );
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
         // Should match nothing since nothing is greater than NaN
-        assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty()));
+        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new()));
 
         // Test IsIn query with mixed float types (Float16, Float32, Float64)
         let query = SargableQuery::IsIn(vec![
@@ -1807,7 +2294,7 @@ mod tests {
         assert_eq!(metadata.get(ZONEMAP_SIZE_META_KEY).unwrap(), "100");
 
         // Read the index file back and check its contents
-        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex");
         assert_eq!(index.zones.len(), 2);
@@ -1903,7 +2390,7 @@ mod tests {
         // 8. IsNull query (no nulls in data, should match nothing)
         let query = SargableQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new()));
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
         // 9. IsIn query: [0, 100, 101, 50]
         let query = SargableQuery::IsIn(vec![
             ScalarValue::Int32(Some(0)),
@@ -1970,7 +2457,7 @@ mod tests {
         log::debug!("Successfully wrote the index file");
 
         // Read the index file back and check its contents
-        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+        let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
             .await
             .expect("Failed to load ZoneMapIndex");
         assert_eq!(index.zones.len(), 3);
@@ -2130,9 +2617,10 @@ mod tests {
             .unwrap();
 
             // Read the index file back and check its contents
-            let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
-                .await
-                .expect("Failed to load ZoneMapIndex");
+            let index =
+                ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
+                    .await
+                    .expect("Failed to load ZoneMapIndex");
             assert_eq!(index.zones.len(), 5);
             assert_eq!(
                 index.zones,
@@ -2338,9 +2826,10 @@ mod tests {
             .unwrap();
 
             // Read the index file back and check its contents
-            let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
-                .await
-                .expect("Failed to load ZoneMapIndex");
+            let index =
+                ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
+                    .await
+                    .expect("Failed to load ZoneMapIndex");
             assert_eq!(index.zones.len(), 3);
             assert_eq!(
                 index.zones,
@@ -2413,9 +2902,10 @@ mod tests {
             .unwrap();
 
             // Read the index file back and check its contents
-            let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
-                .await
-                .expect("Failed to load ZoneMapIndex");
+            let index =
+                ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false)
+                    .await
+                    .expect("Failed to load ZoneMapIndex");
             assert_eq!(index.zones.len(), 3);
             assert_eq!(
                 index.zones,
@@ -2609,9 +3099,11 @@ mod tests {
             zones,
             data_type: DataType::Utf8,
             rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            use_seeds: false,
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix query for "foo"
@@ -2681,9 +3173,11 @@ mod tests {
             zones,
             data_type: DataType::Utf8,
             rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            use_seeds: false,
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix "test"
@@ -2748,9 +3242,11 @@ mod tests {
             zones,
             data_type: DataType::LargeUtf8,
             rows_per_zone: ROWS_PER_ZONE_DEFAULT,
+            use_seeds: false,
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix with LargeUtf8
@@ -2803,4 +3299,313 @@ mod tests {
         // All max characters
         assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None);
     }
+
+    // When merging zone map segments, if ANY source segment has null_rows = None
+    // (legacy — null positions unknown), the merged result must also be None.
+    // The bug: any_null_bitmap is set to true as soon as one source has Some(...),
+    // and the None sources are silently skipped.  The merged index then has
+    // null_rows = Some(partial_bitmap), so an IsNull search returns exact results
+    // that only cover the modern segment's nulls — a false negative for the legacy
+    // segment whose null positions were never tracked.
+    #[tokio::test]
+    async fn test_merge_with_legacy_none_segment_not_treated_as_no_nulls() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        use arrow_array::{Int32Array, UInt32Array};
+
+        // Index A: fragment 0, modern — has a complete null bitmap with 2 known null rows.
+        let schema_a = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch_a = RecordBatch::try_new(
+            schema_a,
+            vec![
+                Arc::new(Int32Array::from(vec![Some(1i32)])) as _,
+                Arc::new(Int32Array::from(vec![Some(5i32)])) as _,
+                Arc::new(UInt32Array::from(vec![2u32])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![10u64])) as _,
+            ],
+        )
+        .unwrap();
+        let mut modern_null_rows = RowAddrTreeMap::new();
+        modern_null_rows.insert(3); // frag 0 row 3
+        modern_null_rows.insert(7); // frag 0 row 7
+        let cache = LanceCache::no_cache();
+        let index_a = Arc::new(
+            ZoneMapIndex::try_from_serialized(
+                batch_a,
+                store.clone(),
+                None,
+                &cache,
+                10,
+                Some(modern_null_rows), // modern: complete bitmap
+                false,
+            )
+            .unwrap(),
+        );
+
+        // Index B: fragment 1, legacy — null_rows = None despite null_count = 3.
+        let schema_b = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch_b = RecordBatch::try_new(
+            schema_b,
+            vec![
+                Arc::new(Int32Array::from(vec![Some(10i32)])) as _,
+                Arc::new(Int32Array::from(vec![Some(20i32)])) as _,
+                Arc::new(UInt32Array::from(vec![3u32])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![1u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![10u64])) as _,
+            ],
+        )
+        .unwrap();
+        let index_b = Arc::new(
+            ZoneMapIndex::try_from_serialized(
+                batch_b,
+                store.clone(),
+                None,
+                &cache,
+                10,
+                None, // legacy: null positions unknown
+                false,
+            )
+            .unwrap(),
+        );
+
+        let dest_tmpdir = TempObjDir::default();
+        let dest_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            dest_tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let all_frags = RoaringBitmap::from_iter([0u32, 1]);
+        merge_zonemap_indices(
+            &[index_a.as_ref(), index_b.as_ref()],
+            dest_store.as_ref(),
+            &all_frags,
+        )
+        .await
+        .unwrap();
+
+        let merged = ZoneMapIndex::load(dest_store.clone(), None, &LanceCache::no_cache(), false)
+            .await
+            .unwrap();
+
+        // Index B had null_rows = None, so the merged index cannot know all null positions.
+        // IsNull must NOT return exact — that would be a false negative for fragment 1's nulls.
+        let result = merged
+            .search(&SargableQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+
+        // With the bug: any_null_bitmap=true (from A) → null_rows=Some(A's bitmap only)
+        //              → IsNull returns exact, missing B's unknown nulls ← FALSE NEGATIVE
+        // With the fix: any_null_bitmap=false (because B is None) → null_rows=None
+        //              → IsNull falls through to zone scan → AtMost
+        assert!(
+            !result.is_exact(),
+            "IsNull on a merged index where one source had null_rows=None must not return \
+             exact; the legacy segment had null_count=3 so its nulls exist at unknown positions"
+        );
+    }
+
+    // Writes a zonemap file in the legacy format (no null bitmap global buffer),
+    // simulating an index created before the null bitmap feature was added.
+    async fn write_legacy_zonemap(store: &dyn IndexStore, null_count: u32) {
+        use arrow_array::{Int32Array, UInt32Array};
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![Some(0)])) as _,
+                Arc::new(Int32Array::from(vec![Some(99)])) as _,
+                Arc::new(UInt32Array::from(vec![null_count])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![100u64])) as _,
+            ],
+        )
+        .unwrap();
+        let mut file_schema = schema.as_ref().clone();
+        file_schema
+            .metadata
+            .insert(ZONEMAP_SIZE_META_KEY.to_string(), "8192".to_string());
+        let mut writer = store
+            .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema))
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_legacy_zonemap_no_null_bitmap() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Write a legacy index with one zone that has nulls but no null bitmap.
+        write_legacy_zonemap(store.as_ref(), 10).await;
+
+        let index = ZoneMapIndex::load(store, None, &LanceCache::no_cache(), false)
+            .await
+            .expect("failed to load legacy zonemap");
+
+        assert!(
+            index.null_rows.is_none(),
+            "legacy index should have no null bitmap"
+        );
+
+        // IS NULL should fall back to the zone-scan path and return AtMost, not Exact.
+        let result = index
+            .search(&SargableQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(
+            !result.is_exact(),
+            "IS NULL on a legacy index should not be exact"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_zone_map_seed_writer_round_trip() {
+        use crate::scalar::seed::IndexSeedWriter;
+        use crate::scalar::zonemap::ZoneMapSeedWriter;
+        use arrow_array::{ArrayRef, Int32Array};
+        use datafusion_common::ScalarValue;
+
+        let rows_per_zone = 4u64;
+        let data_type = DataType::Int32;
+        let mut writer = ZoneMapSeedWriter::new("test_col", rows_per_zone, data_type).unwrap();
+
+        // Batch 1: values 0..4 (fills exactly one zone)
+        let batch1: ArrayRef = Arc::new(Int32Array::from_iter_values(0..4));
+        writer.observe_batch(&batch1).unwrap();
+
+        // Batch 2: values 10..14 (fills a second zone exactly)
+        let batch2: ArrayRef = Arc::new(Int32Array::from_iter_values(10..14));
+        writer.observe_batch(&batch2).unwrap();
+
+        // Batch 3: values 20..22 (partial final zone)
+        let batch3: ArrayRef = Arc::new(Int32Array::from_iter_values(20..22));
+        writer.observe_batch(&batch3).unwrap();
+
+        let bytes = writer.finish().unwrap().expect("should produce bytes");
+
+        // Check schema metadata key/value format
+        assert_eq!(writer.schema_metadata_key(), "lance.seed.test_col");
+        let meta_val = writer.schema_metadata_value(3);
+        assert_eq!(meta_val, "3:4");
+
+        // Deserialize and verify
+        let zones = ZoneMapSeedWriter::deserialize_seed(42, &bytes, rows_per_zone).unwrap();
+        assert_eq!(zones.len(), 3, "expected 3 zones");
+
+        // Zone 0: values 0..4 -> min=0, max=3
+        assert_eq!(zones[0].bound.fragment_id, 42);
+        assert_eq!(zones[0].bound.start, 0);
+        assert_eq!(zones[0].min, ScalarValue::Int32(Some(0)));
+        assert_eq!(zones[0].max, ScalarValue::Int32(Some(3)));
+        assert_eq!(zones[0].null_count, 0);
+
+        // Zone 1: values 10..14 -> min=10, max=13
+        assert_eq!(zones[1].bound.start, 4);
+        assert_eq!(zones[1].min, ScalarValue::Int32(Some(10)));
+        assert_eq!(zones[1].max, ScalarValue::Int32(Some(13)));
+
+        // Zone 2: values 20..22 -> min=20, max=21, partial zone of 2 rows
+        assert_eq!(zones[2].bound.start, 8);
+        assert_eq!(
+            zones[2].bound.length, 2,
+            "partial zone length must be exact"
+        );
+        assert_eq!(zones[2].min, ScalarValue::Int32(Some(20)));
+        assert_eq!(zones[2].max, ScalarValue::Int32(Some(21)));
+
+        // Full zones must have the full rows_per_zone length
+        assert_eq!(zones[0].bound.length, rows_per_zone as usize);
+        assert_eq!(zones[1].bound.length, rows_per_zone as usize);
+    }
+
+    #[tokio::test]
+    async fn test_zone_map_seed_writer_spanning_batches() {
+        use crate::scalar::seed::IndexSeedWriter;
+        use crate::scalar::zonemap::ZoneMapSeedWriter;
+        use arrow_array::{ArrayRef, Int32Array};
+        use datafusion_common::ScalarValue;
+
+        let rows_per_zone = 5u64;
+        let data_type = DataType::Int32;
+        let mut writer = ZoneMapSeedWriter::new("val", rows_per_zone, data_type).unwrap();
+
+        // Single batch with 12 values -> should produce 2 complete zones + 1 partial
+        let batch: ArrayRef = Arc::new(Int32Array::from_iter_values(0..12));
+        writer.observe_batch(&batch).unwrap();
+
+        let bytes = writer.finish().unwrap().expect("should produce bytes");
+        let zones = ZoneMapSeedWriter::deserialize_seed(1, &bytes, rows_per_zone).unwrap();
+        assert_eq!(
+            zones.len(),
+            3,
+            "expected 3 zones from 12 rows with zone size 5"
+        );
+
+        // Zone 0: rows 0..5
+        assert_eq!(zones[0].bound.length, 5);
+        assert_eq!(zones[0].min, ScalarValue::Int32(Some(0)));
+        assert_eq!(zones[0].max, ScalarValue::Int32(Some(4)));
+        // Zone 1: rows 5..10
+        assert_eq!(zones[1].bound.length, 5);
+        assert_eq!(zones[1].min, ScalarValue::Int32(Some(5)));
+        assert_eq!(zones[1].max, ScalarValue::Int32(Some(9)));
+        // Zone 2: rows 10..12 (partial)
+        assert_eq!(zones[2].bound.length, 2, "partial zone length must be 2");
+        assert_eq!(zones[2].min, ScalarValue::Int32(Some(10)));
+        assert_eq!(zones[2].max, ScalarValue::Int32(Some(11)));
+    }
+
+    #[tokio::test]
+    async fn test_zone_map_seed_writer_empty() {
+        use crate::scalar::seed::IndexSeedWriter;
+        use crate::scalar::zonemap::ZoneMapSeedWriter;
+
+        let mut writer = ZoneMapSeedWriter::new("col", 8, DataType::Int32).unwrap();
+        let result = writer.finish().unwrap();
+        assert!(result.is_none(), "empty fragment should return None");
+    }
 }
diff --git a/rust/lance-index/src/vector.rs b/rust/lance-index/src/vector.rs
index 76b9ccc7a51..1192b2f5afb 100644
--- a/rust/lance-index/src/vector.rs
+++ b/rust/lance-index/src/vector.rs
@@ -85,8 +85,9 @@ pub const DEFAULT_QUERY_PARALLELISM: i32 = 0;
 
 /// Controls the speed / accuracy tradeoff for approximate vector search.
 ///
-/// This currently only affects RQ-quantized vector indexes, such as IVF_RQ.
-/// Other index types ignore this setting.
+/// This currently affects RQ-quantized vector indexes (such as IVF_RQ) and
+/// prefiltered search on HNSW sub-indexes, where `Fast` enables the ACORN
+/// traversal. Other index types ignore this setting.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ApproxMode {
     /// Prefer lower query latency, which can reduce recall.
@@ -354,6 +355,54 @@ pub trait VectorIndex: Send + Sync + std::fmt::Debug + Index {
         )))
     }
 
+    /// Whether this index can search multiple query vectors in a single pass
+    /// via [`VectorIndex::search_partitions_batch`], reading each partition's
+    /// storage once and scoring every query that probes it.
+    ///
+    /// Defaults to `false`; callers should fall back to repeated single-query
+    /// search for indices that return `false`.
+    fn supports_batch_partition_search(&self) -> bool {
+        false
+    }
+
+    /// Search a batch of query vectors against a shared set of partitions.
+    ///
+    /// `query.key` holds all query vectors concatenated (length
+    /// `query_count * dim`, where `query_count == partitions_per_query.len()`).
+    /// `partitions_per_query[i]` / `q_c_dists_per_query[i]` are the ranked
+    /// partition ids and query-to-centroid distances for query `i`.
+    ///
+    /// Returns one [RecordBatch] per query (in query order) with the
+    /// [`VECTOR_RESULT_SCHEMA`] (`_distance`, `_rowid`) and at most `query.k`
+    /// rows each. Implementations should read each distinct partition's storage
+    /// only once and score every query assigned to it against the loaded data.
+    ///
+    /// The default implementation returns an error; callers must gate on
+    /// [`VectorIndex::supports_batch_partition_search`].
+    #[allow(clippy::too_many_arguments)]
+    async fn search_partitions_batch(
+        self: Arc<Self>,
+        query: Query,
+        partitions_per_query: Vec<Arc<UInt32Array>>,
+        q_c_dists_per_query: Vec<Arc<Float32Array>>,
+        pre_filter: Arc<dyn PreFilter>,
+        metrics: Arc<dyn MetricsCollector>,
+    ) -> Result<Vec<RecordBatch>>
+    where
+        Self: 'static,
+    {
+        let _ = (
+            query,
+            partitions_per_query,
+            q_c_dists_per_query,
+            pre_filter,
+            metrics,
+        );
+        Err(Error::not_supported(
+            "batch partition search is not supported for this index",
+        ))
+    }
+
     /// If the index is loadable by IVF, so it can be a sub-index that
     /// is loaded on demand by IVF.
     fn is_loadable(&self) -> bool;
diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs
index 7bcc2526b43..72fa8d4b056 100644
--- a/rust/lance-index/src/vector/bq/storage.rs
+++ b/rust/lance-index/src/vector/bq/storage.rs
@@ -3930,7 +3930,7 @@ mod tests {
             .into_iter()
             .map(|(id, dist)| (id as u64, dist))
             .collect::<Vec<_>>();
-        expected.sort_by(|left, right| left.0.cmp(&right.0));
+        expected.sort_by_key(|left| left.0);
 
         let mut heap = BinaryHeap::with_capacity(k);
         let mut distances = Vec::new();
@@ -3952,7 +3952,7 @@ mod tests {
             .into_iter()
             .map(|node| (node.id, node.dist.0))
             .collect::<Vec<_>>();
-        actual.sort_by(|left, right| left.0.cmp(&right.0));
+        actual.sort_by_key(|left| left.0);
 
         assert_eq!(actual.len(), expected.len());
         for ((actual_id, actual_dist), (expected_id, expected_dist)) in
diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs
index 70371ad4794..19aa3fa177d 100755
--- a/rust/lance-index/src/vector/distributed/index_merger.rs
+++ b/rust/lance-index/src/vector/distributed/index_merger.rs
@@ -112,6 +112,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
                 return false;
             }
             for i in 0..av.len() {
+                if av[i].is_nan() || bv[i].is_nan() {
+                    return false;
+                }
                 if (av[i] - bv[i]).abs() > tol {
                     return false;
                 }
@@ -127,6 +130,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
                 return false;
             }
             for i in 0..av.len() {
+                if av[i].is_nan() || bv[i].is_nan() {
+                    return false;
+                }
                 if (av[i] - bv[i]).abs() > tol as f64 {
                     return false;
                 }
@@ -144,6 +150,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
             for i in 0..av.len() {
                 let da = av[i].to_f32();
                 let db = bv[i].to_f32();
+                if da.is_nan() || db.is_nan() {
+                    return false;
+                }
                 if (da - db).abs() > tol {
                     return false;
                 }
@@ -154,6 +163,63 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
     }
 }
 
+fn ensure_fixed_size_list_compatible(
+    what: &str,
+    reference: &FixedSizeListArray,
+    candidate: &FixedSizeListArray,
+) -> Result<()> {
+    if !fixed_size_list_equal(reference, candidate) {
+        const TOL: f32 = 1e-5;
+        if !fixed_size_list_almost_equal(reference, candidate, TOL) {
+            return Err(Error::index(format!("{what} mismatch across shards")));
+        }
+        log::warn!("{what} differs within tolerance; proceeding with first shard value");
+    }
+    Ok(())
+}
+
+async fn try_read_ivf_proto(reader: &V2Reader) -> Result<Option<pb::Ivf>> {
+    let Some(ivf_idx) = reader.metadata().file_schema.metadata.get(IVF_METADATA_KEY) else {
+        return Ok(None);
+    };
+    let ivf_idx = ivf_idx
+        .parse()
+        .map_err(|_| Error::index("IVF index parse error".to_string()))?;
+    let bytes = reader.read_global_buffer(ivf_idx).await?;
+    Ok(Some(pb::Ivf::decode(bytes)?))
+}
+
+fn ivf_centroids_from_proto(ivf: &pb::Ivf) -> Result<Option<FixedSizeListArray>> {
+    ivf.centroids_tensor
+        .as_ref()
+        .map(FixedSizeListArray::try_from)
+        .transpose()
+}
+
+async fn open_sibling_index_reader(
+    object_store: &lance_io::object_store::ObjectStore,
+    sched: &Arc<ScanScheduler>,
+    idx_path: &object_store::path::Path,
+) -> Result<Option<V2Reader>> {
+    if !object_store.exists(idx_path).await? {
+        return Ok(None);
+    }
+
+    let fh = sched
+        .open_file(idx_path, &CachedFileSize::unknown())
+        .await?;
+    Ok(Some(
+        V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &lance_core::cache::LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await?,
+    ))
+}
+
 /// Initialize schema-level metadata on a writer for a given storage.
 ///
 /// It writes the distance type and the storage metadata (as a vector payload),
@@ -771,6 +837,12 @@ pub async fn merge_partial_vector_auxiliary_files(
         )
         .await?;
         let meta = reader.metadata();
+        let idx_path = aux
+            .parent()
+            .unwrap_or_default()
+            .join(crate::INDEX_FILE_NAME);
+        let mut idx_reader: Option<V2Reader> = None;
+        let mut idx_reader_checked = false;
 
         // Inherit format version from the first shard file
         if format_version.is_none() {
@@ -795,45 +867,33 @@ pub async fn merge_partial_vector_auxiliary_files(
         // Detect index type (first iteration only)
         if detected_index_type.is_none() {
             // Try to derive precise type from sibling partial index.idx metadata if available
-            let idx_path = aux
-                .parent()
-                .unwrap_or_default()
-                .join(crate::INDEX_FILE_NAME);
-            if object_store.exists(&idx_path).await.unwrap_or(false) {
-                let fh2 = sched
-                    .open_file(&idx_path, &CachedFileSize::unknown())
-                    .await?;
-                let idx_reader = V2Reader::try_open(
-                    fh2,
-                    None,
-                    Arc::default(),
-                    &lance_core::cache::LanceCache::no_cache(),
-                    V2ReaderOptions::default(),
-                )
-                .await?;
-                if let Some(idx_meta_json) = idx_reader
+            if !idx_reader_checked {
+                idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?;
+                idx_reader_checked = true;
+            }
+            if let Some(idx_reader) = idx_reader.as_ref()
+                && let Some(idx_meta_json) = idx_reader
                     .metadata()
                     .file_schema
                     .metadata
                     .get(INDEX_METADATA_SCHEMA_KEY)
-                {
-                    let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?;
-                    detected_index_type = Some(match idx_meta.index_type.as_str() {
-                        "IVF_FLAT" => SupportedIvfIndexType::IvfFlat,
-                        "IVF_PQ" => SupportedIvfIndexType::IvfPq,
-                        "IVF_SQ" => SupportedIvfIndexType::IvfSq,
-                        "IVF_RQ" => SupportedIvfIndexType::IvfRq,
-                        "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat,
-                        "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq,
-                        "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq,
-                        other => {
-                            return Err(Error::index(format!(
-                                "Unsupported index type in shard index.idx: {}",
-                                other
-                            )));
-                        }
-                    });
-                }
+            {
+                let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?;
+                detected_index_type = Some(match idx_meta.index_type.as_str() {
+                    "IVF_FLAT" => SupportedIvfIndexType::IvfFlat,
+                    "IVF_PQ" => SupportedIvfIndexType::IvfPq,
+                    "IVF_SQ" => SupportedIvfIndexType::IvfSq,
+                    "IVF_RQ" => SupportedIvfIndexType::IvfRq,
+                    "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat,
+                    "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq,
+                    "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq,
+                    other => {
+                        return Err(Error::index(format!(
+                            "Unsupported index type in shard index.idx: {}",
+                            other
+                        )));
+                    }
+                });
             }
             // Fallback: infer from auxiliary schema
             if detected_index_type.is_none() {
@@ -843,35 +903,52 @@ pub async fn merge_partial_vector_auxiliary_files(
         }
 
         // Read IVF lengths from global buffer
-        let ivf_idx: u32 = reader
-            .metadata()
-            .file_schema
-            .metadata
-            .get(IVF_METADATA_KEY)
-            .ok_or_else(|| Error::index("IVF meta missing".to_string()))?
-            .parse()
-            .map_err(|_| Error::index("IVF index parse error".to_string()))?;
-        let bytes = reader.read_global_buffer(ivf_idx).await?;
-        let pb_ivf: pb::Ivf = prost::Message::decode(bytes)?;
+        let pb_ivf = try_read_ivf_proto(&reader)
+            .await?
+            .ok_or_else(|| Error::index("IVF meta missing".to_string()))?;
         let lengths = pb_ivf.lengths.clone();
         let nlist = lengths.len();
 
+        let mut current_centroids = ivf_centroids_from_proto(&pb_ivf)?;
+        if current_centroids.is_none() {
+            if !idx_reader_checked {
+                idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?;
+            }
+            if let Some(idx_reader) = idx_reader.as_ref()
+                && let Some(index_ivf) = try_read_ivf_proto(idx_reader).await?
+            {
+                current_centroids = ivf_centroids_from_proto(&index_ivf)?;
+            }
+        }
         if nlist_opt.is_none() {
             nlist_opt = Some(nlist);
             accumulated_lengths = vec![0; nlist];
-            // Try load centroids tensor if present
-            if let Some(tensor) = pb_ivf.centroids_tensor.as_ref() {
-                let arr = FixedSizeListArray::try_from(tensor)?;
-                first_centroids = Some(arr.clone());
+            if let Some(arr) = current_centroids {
                 let d0 = arr.value_length() as usize;
                 if dim.is_none() {
                     dim = Some(d0);
                 }
+                first_centroids = Some(arr);
             }
         } else if nlist_opt.as_ref().map(|v| *v != nlist).unwrap_or(false) {
             return Err(Error::index(
                 "IVF partition count mismatch across shards".to_string(),
             ));
+        } else {
+            match (&first_centroids, &current_centroids) {
+                (Some(reference), Some(candidate)) => {
+                    ensure_fixed_size_list_compatible("IVF centroids", reference, candidate)?;
+                }
+                (Some(_), None) => {
+                    return Err(Error::index("IVF centroids missing from shard".to_string()));
+                }
+                (None, Some(_)) => {
+                    return Err(Error::index(
+                        "IVF centroids missing from first shard".to_string(),
+                    ));
+                }
+                (None, None) => {}
+            }
         }
 
         // Handle logic based on detected index type
@@ -985,6 +1062,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                     rq_meta_parsed.parse_buffer(rotate_mat_bytes)?;
                 }
                 validate_rq_num_bits(rq_meta_parsed.num_bits)?;
+                if rq_meta_parsed.packed {
+                    return Err(Error::index(format!(
+                        "Distributed RQ merge: source shard {idx} stores packed RQ codes; expected row-major distributed shard"
+                    )));
+                }
 
                 let d0 = rq_meta_parsed.rotated_dim();
                 if d0 == 0 {
@@ -1001,7 +1083,9 @@ pub async fn merge_partial_vector_auxiliary_files(
                 if let Some(existing_rq) = rq_meta.as_ref()
                     && (existing_rq.code_dim != rq_meta_parsed.code_dim
                         || existing_rq.num_bits != rq_meta_parsed.num_bits
-                        || existing_rq.rotation_type != rq_meta_parsed.rotation_type)
+                        || existing_rq.rotation_type != rq_meta_parsed.rotation_type
+                        || existing_rq.query_estimator != rq_meta_parsed.query_estimator
+                        || existing_rq.fast_rotation_signs != rq_meta_parsed.fast_rotation_signs)
                 {
                     return Err(Error::index(format!(
                         "Distributed RQ merge: structural mismatch across shards; first(code_dim={}, num_bits={}, rotation_type={:?}), current(code_dim={}, num_bits={}, rotation_type={:?})",
@@ -1013,6 +1097,24 @@ pub async fn merge_partial_vector_auxiliary_files(
                         rq_meta_parsed.rotation_type
                     )));
                 }
+                if let Some(existing_rq) = rq_meta.as_ref() {
+                    match (&existing_rq.rotate_mat, &rq_meta_parsed.rotate_mat) {
+                        (Some(reference), Some(candidate)) => {
+                            ensure_fixed_size_list_compatible(
+                                "RQ rotation matrix",
+                                reference,
+                                candidate,
+                            )?;
+                        }
+                        (Some(_), None) | (None, Some(_)) => {
+                            return Err(Error::index(
+                                "Distributed RQ merge: rotation matrix mismatch across shards"
+                                    .to_string(),
+                            ));
+                        }
+                        (None, None) => {}
+                    }
+                }
                 if rq_meta.is_none() {
                     rq_meta = Some(rq_meta_parsed.clone());
                 }
@@ -1061,6 +1163,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                 };
                 let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json)
                     .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?;
+                if pm.transposed {
+                    return Err(Error::index(format!(
+                        "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard"
+                    )));
+                }
                 // Load codebook from global buffer if not present
                 if pm.codebook.is_none() {
                     let tensor_bytes = reader
@@ -1100,18 +1207,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                         .codebook
                         .as_ref()
                         .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?;
-                    if !fixed_size_list_equal(existing_cb, current_cb) {
-                        const TOL: f32 = 1e-5;
-                        if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) {
-                            return Err(Error::index(
-                                "PQ codebook content mismatch across shards".to_string(),
-                            ));
-                        } else {
-                            log::warn!(
-                                "PQ codebook differs within tolerance; proceeding with first shard codebook"
-                            );
-                        }
-                    }
+                    ensure_fixed_size_list_compatible(
+                        "PQ codebook content",
+                        existing_cb,
+                        current_cb,
+                    )?;
                 }
                 if pq_meta.is_none() {
                     pq_meta = Some(pm.clone());
@@ -1222,6 +1322,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                 };
                 let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json)
                     .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?;
+                if pm.transposed {
+                    return Err(Error::index(format!(
+                        "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard"
+                    )));
+                }
                 if pm.codebook.is_none() {
                     let tensor_bytes = reader
                         .read_global_buffer(pm.codebook_position as u32)
@@ -1260,18 +1365,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                         .codebook
                         .as_ref()
                         .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?;
-                    if !fixed_size_list_equal(existing_cb, current_cb) {
-                        const TOL: f32 = 1e-5;
-                        if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) {
-                            return Err(Error::index(
-                                "PQ codebook content mismatch across shards".to_string(),
-                            ));
-                        } else {
-                            log::warn!(
-                                "PQ codebook differs within tolerance; proceeding with first shard codebook"
-                            );
-                        }
-                    }
+                    ensure_fixed_size_list_compatible(
+                        "PQ codebook content",
+                        existing_cb,
+                        current_cb,
+                    )?;
                 }
                 if pq_meta.is_none() {
                     pq_meta = Some(pm.clone());
@@ -1976,6 +2074,7 @@ mod tests {
         base_row_id: u64,
         distance_type: DistanceType,
         codebook: &FixedSizeListArray,
+        transposed: bool,
     ) -> Result<usize> {
         let num_bytes = if nbits == 4 {
             // Two 4-bit codes per byte.
@@ -2014,7 +2113,7 @@ mod tests {
             dimension,
             codebook: Some(codebook.clone()),
             codebook_tensor: Vec::new(),
-            transposed: true,
+            transposed,
         };
 
         let codebook_tensor: pb::Tensor = pb::Tensor::try_from(codebook)?;
@@ -2225,6 +2324,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2239,6 +2339,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2319,6 +2420,66 @@ mod tests {
         assert!(fixed_size_list_equal(&codebook, &merged_codebook));
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_pq_rejects_transposed_source_shard() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_pq_transposed");
+
+        let partial0 = index_dir.clone().join("partial_0");
+        let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let lengths = vec![2_u32, 1_u32];
+
+        let nbits = 4_u32;
+        let num_sub_vectors = 2_usize;
+        let dimension = 8_usize;
+        let num_centroids = 1_usize << nbits;
+        let num_codebook_vectors = num_centroids * num_sub_vectors;
+        let total_values = num_codebook_vectors * dimension;
+        let values = Float32Array::from_iter((0..total_values).map(|v| v as f32));
+        let codebook = FixedSizeListArray::try_new_from_values(values, dimension as i32).unwrap();
+
+        write_pq_partial_aux(
+            &object_store,
+            &aux0,
+            nbits,
+            num_sub_vectors,
+            dimension,
+            &lengths,
+            0,
+            DistanceType::L2,
+            &codebook,
+            true,
+        )
+        .await
+        .unwrap();
+
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            std::slice::from_ref(&aux0),
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
+        match res {
+            Err(Error::Index { message, .. }) => {
+                assert!(
+                    message.contains("source shard 0"),
+                    "unexpected message: {}",
+                    message
+                );
+                assert!(
+                    message.contains("transposed PQ codes"),
+                    "unexpected message: {}",
+                    message
+                );
+            }
+            other => panic!(
+                "expected Error::Index for transposed PQ source shard, got {:?}",
+                other
+            ),
+        }
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_rq_success() {
         let object_store = ObjectStore::memory();
@@ -2455,6 +2616,64 @@ mod tests {
         assert_eq!(total_rows, expected_total);
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_rq_rejects_packed_source_shard() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_rq_packed");
+
+        let partial0 = index_dir.clone().join("partial_0");
+        let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let lengths = vec![2_u32, 1_u32];
+
+        let rq_meta = RabitQuantizationMetadata {
+            rotate_mat: None,
+            rotate_mat_position: None,
+            fast_rotation_signs: Some(vec![0xAA; 2]),
+            rotation_type: RQRotationType::Fast,
+            code_dim: 16,
+            num_bits: 1,
+            packed: true,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+
+        write_rq_partial_aux(
+            &object_store,
+            &aux0,
+            &rq_meta,
+            &lengths,
+            0,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            std::slice::from_ref(&aux0),
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
+        match res {
+            Err(Error::Index { message, .. }) => {
+                assert!(
+                    message.contains("source shard 0"),
+                    "unexpected message: {}",
+                    message
+                );
+                assert!(
+                    message.contains("packed RQ codes"),
+                    "unexpected message: {}",
+                    message
+                );
+            }
+            other => panic!(
+                "expected Error::Index for packed RQ source shard, got {:?}",
+                other
+            ),
+        }
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_rq_multi_bit_preserves_split_columns() {
         let object_store = ObjectStore::memory();
@@ -2612,6 +2831,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook0,
+            false,
         )
         .await
         .unwrap();
@@ -2626,6 +2846,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook1,
+            false,
         )
         .await
         .unwrap();
@@ -2691,6 +2912,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2706,6 +2928,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
diff --git a/rust/lance-index/src/vector/flat/index.rs b/rust/lance-index/src/vector/flat/index.rs
index cc6f6d021eb..89c28a6ffd7 100644
--- a/rust/lance-index/src/vector/flat/index.rs
+++ b/rust/lance-index/src/vector/flat/index.rs
@@ -570,7 +570,7 @@ mod tests {
             .zip(dists.values().iter())
             .map(|(row_id, dist)| (*row_id, *dist))
             .collect::<Vec<_>>();
-        results.sort_by(|left, right| left.0.cmp(&right.0));
+        results.sort_by_key(|left| left.0);
         results
     }
 
@@ -579,7 +579,7 @@ mod tests {
             .into_iter()
             .map(|node| (node.id, node.dist.0))
             .collect::<Vec<_>>();
-        results.sort_by(|left, right| left.0.cmp(&right.0));
+        results.sort_by_key(|left| left.0);
         results
     }
 
diff --git a/rust/lance-index/src/vector/flat/storage.rs b/rust/lance-index/src/vector/flat/storage.rs
index c3ec30d5086..ff8756cbe41 100644
--- a/rust/lance-index/src/vector/flat/storage.rs
+++ b/rust/lance-index/src/vector/flat/storage.rs
@@ -134,7 +134,7 @@ impl VectorStore for FlatFloatStorage {
 
     fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result<Self> {
         // TODO: use chunked storage
-        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
+        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?;
         let mut storage = self.clone();
         storage.row_ids = Arc::new(
             new_batch
@@ -296,7 +296,7 @@ impl VectorStore for FlatBinStorage {
 
     fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result<Self> {
         // TODO: use chunked storage
-        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
+        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?;
         let mut storage = self.clone();
         storage.row_ids = Arc::new(
             new_batch
diff --git a/rust/lance-index/src/vector/flat/transform.rs b/rust/lance-index/src/vector/flat/transform.rs
index 75a465ce262..f9fdca0819c 100644
--- a/rust/lance-index/src/vector/flat/transform.rs
+++ b/rust/lance-index/src/vector/flat/transform.rs
@@ -26,7 +26,7 @@ impl FlatTransformer {
 
 impl Transformer for FlatTransformer {
     #[instrument(name = "FlatTransformer::transform", level = "debug", skip_all)]
-    fn transform(&self, batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    fn transform(&self, batch: &RecordBatch) -> lance_core::Result<RecordBatch> {
         let input_arr = batch
             .column_by_name(&self.input_column)
             .ok_or(Error::index(format!(
diff --git a/rust/lance-index/src/vector/graph.rs b/rust/lance-index/src/vector/graph.rs
index 097aa064d67..0bba63a6bf6 100644
--- a/rust/lance-index/src/vector/graph.rs
+++ b/rust/lance-index/src/vector/graph.rs
@@ -5,7 +5,7 @@
 //!
 
 use std::cmp::Reverse;
-use std::collections::BinaryHeap;
+use std::collections::{BinaryHeap, VecDeque};
 use std::sync::Arc;
 
 use arrow_schema::{DataType, Field};
@@ -546,6 +546,152 @@ pub fn beam_search_borrowed(
     results.into_sorted_vec()
 }
 
+/// Number of mask-passing nodes used to seed [beam_search_acorn]'s frontier.
+const ACORN_SEED_COUNT: usize = 16;
+
+/// Cap on starved-frontier waypoint expansions in [beam_search_acorn],
+/// as a multiple of `ef`.
+const ACORN_BRIDGE_BUDGET_FACTOR: usize = 4;
+
+/// Beam search over the mask-passing subgraph (ACORN-1).
+///
+/// Only nodes in `bitset` get distances. A filtered-out neighbor contributes
+/// its own neighbors instead, expanded once via `expanded`. Deeper masked
+/// chains are crossed through unscored waypoints under a budget. The frontier
+/// starts from the entry point plus mask-sampled seeds. May return fewer than
+/// `min(ef, passing)` results if the budget runs out, so callers needing a
+/// guarantee must check the count.
+#[allow(clippy::too_many_arguments)]
+pub fn beam_search_acorn(
+    graph: &impl BorrowingGraph,
+    ep: &OrderedNode,
+    params: &HnswQueryParams,
+    dist_calc: &impl DistCalculator,
+    bitset: &Visited,
+    prefetch_distance: Option<usize>,
+    visited: &mut Visited,
+    expanded: &mut Visited,
+) -> Vec<OrderedNode> {
+    let ef = params.ef;
+    let lower_bound: OrderedFloat = params.lower_bound.unwrap_or(f32::MIN).into();
+    let upper_bound: OrderedFloat = params.upper_bound.unwrap_or(f32::MAX).into();
+    let passing_total = bitset.count_ones();
+    let mut candidates = BinaryHeap::with_capacity(ef);
+    let mut results = BinaryHeap::with_capacity(ef);
+    // collected per node before scoring so prefetch targets are the ids
+    // that actually get distances
+    let mut passing: Vec<u32> = Vec::with_capacity(64);
+    // masked nodes seen two hops out, expandable if the frontier starves,
+    // deduped against `expanded` at pop rather than at push
+    let mut waypoints: VecDeque<u32> = VecDeque::new();
+    let mut bridge_budget = ACORN_BRIDGE_BUDGET_FACTOR * ef;
+
+    // the entry point seeds the traversal even if it fails the mask
+    visited.insert(ep.id);
+    candidates.push(Reverse(ep.clone()));
+    if bitset.contains(ep.id) && ep.dist >= lower_bound && ep.dist < upper_bound {
+        results.push(ep.clone());
+    }
+
+    let stride = (passing_total / ACORN_SEED_COUNT).max(1);
+    for seed in bitset.iter_ones().step_by(stride).take(ACORN_SEED_COUNT) {
+        let seed = seed as u32;
+        if visited.contains(seed) {
+            continue;
+        }
+        visited.insert(seed);
+        let dist: OrderedFloat = dist_calc.distance(seed).into();
+        if dist >= lower_bound && dist < upper_bound {
+            push_result(&mut results, (dist, seed).into(), ef);
+        }
+        candidates.push(Reverse((dist, seed).into()));
+    }
+
+    loop {
+        let Some(Reverse(current)) = candidates.pop() else {
+            // frontier starved: burn bridge budget through masked waypoints
+            // until a new passing node is found
+            if results.len() >= ef.min(passing_total) {
+                break;
+            }
+            let mut found = false;
+            while let Some(waypoint) = waypoints.pop_front() {
+                if bridge_budget == 0 {
+                    break;
+                }
+                if expanded.contains(waypoint) {
+                    continue;
+                }
+                expanded.insert(waypoint);
+                bridge_budget -= 1;
+                for &neighbor in graph.neighbors(waypoint) {
+                    if bitset.contains(neighbor) {
+                        if !visited.contains(neighbor) {
+                            visited.insert(neighbor);
+                            let dist: OrderedFloat = dist_calc.distance(neighbor).into();
+                            if dist >= lower_bound && dist < upper_bound {
+                                push_result(&mut results, (dist, neighbor).into(), ef);
+                            }
+                            candidates.push(Reverse((dist, neighbor).into()));
+                            found = true;
+                        }
+                    } else if !expanded.contains(neighbor) {
+                        waypoints.push_back(neighbor);
+                    }
+                }
+                if found {
+                    break;
+                }
+            }
+            if !found {
+                break;
+            }
+            continue;
+        };
+        if current.dist > furthest_distance(&results) && results.len() == ef {
+            break;
+        }
+
+        passing.clear();
+        for &neighbor in graph.neighbors(current.id) {
+            if bitset.contains(neighbor) {
+                if !visited.contains(neighbor) {
+                    visited.insert(neighbor);
+                    passing.push(neighbor);
+                }
+            } else if !expanded.contains(neighbor) {
+                expanded.insert(neighbor);
+                for &second_hop in graph.neighbors(neighbor) {
+                    if bitset.contains(second_hop) {
+                        if !visited.contains(second_hop) {
+                            visited.insert(second_hop);
+                            passing.push(second_hop);
+                        }
+                    } else if !expanded.contains(second_hop) {
+                        waypoints.push_back(second_hop);
+                    }
+                }
+            }
+        }
+
+        process_neighbors_with_look_ahead(
+            &passing,
+            |node| {
+                let dist: OrderedFloat = dist_calc.distance(node).into();
+                if dist <= furthest_distance(&results) || results.len() < ef {
+                    if dist >= lower_bound && dist < upper_bound {
+                        push_result(&mut results, (dist, node).into(), ef);
+                    }
+                    candidates.push(Reverse((dist, node).into()));
+                }
+            },
+            prefetch_distance,
+            dist_calc,
+        );
+    }
+    results.into_sorted_vec()
+}
+
 /// Greedy search over a graph
 ///
 /// This searches for only one result, only used for finding the entry point
@@ -617,4 +763,96 @@ pub fn greedy_search_borrowed(
 }
 
 #[cfg(test)]
-mod tests {}
+mod tests {
+    use super::*;
+
+    struct ChainGraph {
+        neighbors: Vec<Vec<u32>>,
+    }
+
+    impl BorrowingGraph for ChainGraph {
+        fn len(&self) -> usize {
+            self.neighbors.len()
+        }
+
+        fn neighbors(&self, key: u32) -> &[u32] {
+            &self.neighbors[key as usize]
+        }
+    }
+
+    struct ZeroDistance;
+
+    impl DistCalculator for ZeroDistance {
+        fn distance(&self, _id: u32) -> f32 {
+            0.0
+        }
+
+        fn distance_all(&self, _k_hint: usize) -> Vec<f32> {
+            Vec::new()
+        }
+    }
+
+    /// Passing components joined only through chains of two masked nodes
+    /// must still all be found (from review: without waypoint expansion
+    /// only the seeded nodes return).
+    #[test]
+    fn test_acorn_reaches_across_masked_chains() {
+        const PASSING_COUNT: usize = 20;
+        const FAILING_COUNT: usize = (PASSING_COUNT - 1) * 2;
+        let mut neighbors = vec![Vec::new(); PASSING_COUNT + FAILING_COUNT];
+        for index in 0..PASSING_COUNT - 1 {
+            let left = index as u32;
+            let first_failing = (PASSING_COUNT + index * 2) as u32;
+            let second_failing = first_failing + 1;
+            let right = left + 1;
+
+            neighbors[left as usize].push(first_failing);
+            neighbors[first_failing as usize].extend([left, second_failing]);
+            neighbors[second_failing as usize].extend([first_failing, right]);
+            neighbors[right as usize].push(second_failing);
+        }
+        let graph = ChainGraph { neighbors };
+        let params = HnswQueryParams {
+            ef: 30,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+            use_acorn: false,
+        };
+        let entry = OrderedNode::new(0, 0.0.into());
+
+        let mut mask_generator = VisitedGenerator::new(graph.len());
+        let mut mask = mask_generator.generate(graph.len());
+        for id in 0..PASSING_COUNT as u32 {
+            mask.insert(id);
+        }
+
+        let mut acorn_visited_generator = VisitedGenerator::new(graph.len());
+        let mut acorn_expanded_generator = VisitedGenerator::new(graph.len());
+        let acorn_results = beam_search_acorn(
+            &graph,
+            &entry,
+            &params,
+            &ZeroDistance,
+            &mask,
+            None,
+            &mut acorn_visited_generator.generate(graph.len()),
+            &mut acorn_expanded_generator.generate(graph.len()),
+        );
+
+        let mut basic_visited_generator = VisitedGenerator::new(graph.len());
+        let basic_results = beam_search_borrowed(
+            &graph,
+            &entry,
+            &params,
+            &ZeroDistance,
+            Some(&mask),
+            None,
+            &mut basic_visited_generator.generate(graph.len()),
+        );
+
+        assert_eq!(basic_results.len(), PASSING_COUNT);
+        assert_eq!(acorn_results.len(), PASSING_COUNT);
+        assert!(acorn_results.iter().all(|node| mask.contains(node.id)));
+    }
+}
diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs
index cc1ac1abf81..62a6676f70e 100644
--- a/rust/lance-index/src/vector/hnsw/builder.rs
+++ b/rust/lance-index/src/vector/hnsw/builder.rs
@@ -40,10 +40,12 @@ use crate::vector::graph::{
     BorrowingGraph, DISTS_FIELD, Graph, NEIGHBORS_COL, NEIGHBORS_FIELD, OrderedFloat, OrderedNode,
     VisitedGenerator,
 };
-use crate::vector::graph::{Visited, beam_search_borrowed, greedy_search, greedy_search_borrowed};
+use crate::vector::graph::{
+    Visited, beam_search_acorn, beam_search_borrowed, greedy_search, greedy_search_borrowed,
+};
 use crate::vector::storage::{DistCalculator, VectorStore};
 use crate::vector::v3::subindex::IvfSubIndex;
-use crate::vector::{Query, VECTOR_RESULT_SCHEMA};
+use crate::vector::{ApproxMode, Query, VECTOR_RESULT_SCHEMA};
 
 pub const HNSW_METADATA_KEY: &str = "lance:hnsw";
 
@@ -397,6 +399,139 @@ impl HNSW {
         result
     }
 
+    /// Like [Self::search_basic] but the bottom level runs
+    /// [beam_search_acorn], which only scores mask-passing nodes.
+    pub fn search_acorn(
+        &self,
+        query: ArrayRef,
+        k: usize,
+        params: &HnswQueryParams,
+        bitset: &Visited,
+        storage: &impl VectorStore,
+    ) -> Result<Vec<OrderedNode>> {
+        let mut visited_generator = self
+            .inner
+            .visited_generator_queue
+            .pop()
+            .unwrap_or_else(|| VisitedGenerator::new(storage.len()));
+        let mut expanded_generator = self
+            .inner
+            .visited_generator_queue
+            .pop()
+            .unwrap_or_else(|| VisitedGenerator::new(storage.len()));
+
+        let result = self.search_acorn_inner(
+            query,
+            k,
+            params,
+            bitset,
+            &mut visited_generator,
+            &mut expanded_generator,
+            storage,
+            Some(2),
+        );
+
+        // if the queue is full, we just don't push it back, so ignore the error here
+        let _ = self.inner.visited_generator_queue.push(visited_generator);
+        let _ = self.inner.visited_generator_queue.push(expanded_generator);
+        result
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn search_acorn_inner(
+        &self,
+        query: ArrayRef,
+        k: usize,
+        params: &HnswQueryParams,
+        bitset: &Visited,
+        visited_generator: &mut VisitedGenerator,
+        expanded_generator: &mut VisitedGenerator,
+        storage: &impl VectorStore,
+        prefetch_distance: Option<usize>,
+    ) -> Result<Vec<OrderedNode>> {
+        let dist_calc = storage.dist_calculator(query, params.dist_q_c);
+        let entry = self.inner.entry_point;
+        let ep = OrderedNode::new(entry, dist_calc.distance(entry).into());
+
+        let result = match &self.inner.graph {
+            HnswGraph::Built(nodes) => {
+                let nodes = nodes.as_slice();
+                self.run_search_acorn(
+                    ep,
+                    params,
+                    bitset,
+                    visited_generator,
+                    expanded_generator,
+                    storage.len(),
+                    prefetch_distance,
+                    &dist_calc,
+                    |level| ImmutableHnswLevelView::new(level, nodes),
+                    ImmutableHnswBottomView::new(nodes),
+                )
+            }
+            HnswGraph::Loaded(graph) => {
+                let graph = graph.as_ref();
+                self.run_search_acorn(
+                    ep,
+                    params,
+                    bitset,
+                    visited_generator,
+                    expanded_generator,
+                    storage.len(),
+                    prefetch_distance,
+                    &dist_calc,
+                    |level| LoadedHnswLevelView::new(level, graph),
+                    LoadedHnswBottomView::new(graph),
+                )
+            }
+        };
+        Ok(result.into_iter().take(k).collect())
+    }
+
+    /// [Self::run_search] for the ACORN traversal: same level descent, but
+    /// the bottom level runs [beam_search_acorn].
+    #[allow(clippy::too_many_arguments)]
+    fn run_search_acorn<L, B>(
+        &self,
+        ep: OrderedNode,
+        params: &HnswQueryParams,
+        bitset: &Visited,
+        visited_generator: &mut VisitedGenerator,
+        expanded_generator: &mut VisitedGenerator,
+        storage_len: usize,
+        prefetch_distance: Option<usize>,
+        dist_calc: &impl DistCalculator,
+        make_level: impl Fn(u16) -> L,
+        bottom: B,
+    ) -> Vec<OrderedNode>
+    where
+        L: BorrowingGraph,
+        B: BorrowingGraph,
+    {
+        let mut ep = ep;
+        for level in (0..self.max_level()).rev() {
+            let cur_level = make_level(level);
+            ep = greedy_search_borrowed(
+                &cur_level,
+                ep,
+                dist_calc,
+                self.inner.params.prefetch_distance,
+            );
+        }
+        let mut visited = visited_generator.generate(storage_len);
+        let mut expanded = expanded_generator.generate(storage_len);
+        beam_search_acorn(
+            &bottom,
+            &ep,
+            params,
+            dist_calc,
+            bitset,
+            prefetch_distance,
+            &mut visited,
+            &mut expanded,
+        )
+    }
+
     #[instrument(level = "debug", skip(self, storage, query, prefilter_bitset))]
     fn flat_search(
         &self,
@@ -677,6 +812,7 @@ impl HnswBuilder {
                 lower_bound: None,
                 upper_bound: None,
                 dist_q_c: 0.0,
+                use_acorn: false,
             },
             dist_calc,
             None,
@@ -965,6 +1101,7 @@ pub struct HnswQueryParams {
     pub lower_bound: Option<f32>,
     pub upper_bound: Option<f32>,
     pub dist_q_c: f32,
+    pub use_acorn: bool,
 }
 
 impl From<&Query> for HnswQueryParams {
@@ -975,6 +1112,7 @@ impl From<&Query> for HnswQueryParams {
             lower_bound: query.lower_bound,
             upper_bound: query.upper_bound,
             dist_q_c: query.dist_q_c,
+            use_acorn: query.approx_mode == ApproxMode::Fast,
         }
     }
 }
@@ -1151,27 +1289,40 @@ impl IvfSubIndex for HNSW {
             .visited_generator_queue
             .pop()
             .unwrap_or_else(|| VisitedGenerator::new(storage.len()));
-        let prefilter_bitset = if prefilter.is_empty() {
-            None
+        let results = if prefilter.is_empty() {
+            self.search_basic(query, k, &params, None, storage)?
         } else {
+            // the bitset must be moved into a callee on every path so its
+            // borrow of `prefilter_generator` ends before the push below
             let indices = prefilter.filter_row_ids(Box::new(storage.row_ids()));
-            let mut bitset = prefilter_generator.generate(storage.len());
-            for indices in indices {
-                bitset.insert(indices as u32);
+            let mut prefilter_bitset = prefilter_generator.generate(storage.len());
+            for index in indices {
+                prefilter_bitset.insert(index as u32);
+            }
+            let remained = prefilter_bitset.count_ones();
+            if remained == storage.len() {
+                // mask passes every row: same as unfiltered
+                drop(prefilter_bitset);
+                self.search_basic(query, k, &params, None, storage)?
+            } else if remained < self.len() * 10 / 100 {
+                // few matching rows: brute force is cheaper and exact
+                self.flat_search(storage, query, k, prefilter_bitset, &params)
+            } else if params.use_acorn {
+                let acorn_results =
+                    self.search_acorn(query.clone(), k, &params, &prefilter_bitset, storage)?;
+                // under-delivery means the budget ran out on a fragmented
+                // mask, except range-bounded queries which return short
+                // legitimately
+                let bounded = params.lower_bound.is_some() || params.upper_bound.is_some();
+                if !bounded && acorn_results.len() < k.min(remained) {
+                    self.search_basic(query, k, &params, Some(prefilter_bitset), storage)?
+                } else {
+                    drop(prefilter_bitset);
+                    acorn_results
+                }
+            } else {
+                self.search_basic(query, k, &params, Some(prefilter_bitset), storage)?
             }
-            Some(bitset)
-        };
-
-        let remained = prefilter_bitset
-            .as_ref()
-            .map(|b| b.count_ones())
-            .unwrap_or(storage.len());
-        let results = if remained < self.len() * 10 / 100 {
-            let prefilter_bitset =
-                prefilter_bitset.expect("the prefilter bitset must be set for flat search");
-            self.flat_search(storage, query, k, prefilter_bitset, &params)
-        } else {
-            self.search_basic(query, k, &params, prefilter_bitset, storage)?
         };
         // if the queue is full, we just don't push it back, so ignore the error here
         let _ = self.inner.visited_generator_queue.push(prefilter_generator);
@@ -1330,12 +1481,11 @@ mod tests {
     use rstest::rstest;
 
     use super::HnswGraph;
-    use crate::scalar::IndexWriter;
     use crate::vector::storage::{DistCalculator, VectorStore};
     use crate::vector::v3::subindex::IvfSubIndex;
     use crate::vector::{
         flat::storage::{FlatBinStorage, FlatFloatStorage},
-        graph::{DISTS_FIELD, NEIGHBORS_FIELD},
+        graph::{DISTS_FIELD, NEIGHBORS_FIELD, VisitedGenerator},
         hnsw::{
             HNSW, VECTOR_ID_FIELD,
             builder::{HnswBuildParams, HnswQueryParams},
@@ -1375,7 +1525,7 @@ mod tests {
         .unwrap();
         let batch = builder.to_batch().unwrap();
         let metadata = batch.schema_ref().metadata().clone();
-        writer.write_record_batch(batch).await.unwrap();
+        writer.write(&[batch]).await.unwrap();
         writer.finish_with_metadata(&metadata).await.unwrap();
 
         let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None)
@@ -1394,6 +1544,7 @@ mod tests {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let builder_results = builder
             .search_basic(query.clone(), k, &params, None, store.as_ref())
@@ -1436,7 +1587,7 @@ mod tests {
         .unwrap();
         let batch = builder.to_batch().unwrap();
         let metadata = batch.schema_ref().metadata().clone();
-        writer.write_record_batch(batch).await.unwrap();
+        writer.write(&[batch]).await.unwrap();
         writer.finish_with_metadata(&metadata).await.unwrap();
 
         let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None)
@@ -1455,6 +1606,7 @@ mod tests {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let builder_results = builder
             .search_basic(query.clone(), k, &params, None, store.as_ref())
@@ -1511,6 +1663,7 @@ mod tests {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let query = fsl.value(0);
 
@@ -1534,6 +1687,229 @@ mod tests {
         assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})");
     }
 
+    /// Brute-force top-`k` restricted to mask-passing ids.
+    fn brute_force_topk_masked(
+        store: &FlatFloatStorage,
+        query: ArrayRef,
+        k: usize,
+        passes: impl Fn(u32) -> bool,
+    ) -> Vec<u32> {
+        let dist_calc = store.dist_calculator(query, 0.0);
+        let mut matching: Vec<(f32, u32)> = (0..store.len() as u32)
+            .filter(|id| passes(*id))
+            .map(|id| (dist_calc.distance(id), id))
+            .collect();
+        matching.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+        matching.into_iter().take(k).map(|(_, id)| id).collect()
+    }
+
+    /// ACORN returns only mask-passing nodes, searches built and loaded
+    /// graphs identically, and holds recall vs brute force over the mask.
+    #[tokio::test]
+    async fn test_acorn_filtered_search() {
+        const DIM: usize = 32;
+        const TOTAL: usize = 2048;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+        let loaded = HNSW::load(builder.to_batch().unwrap()).unwrap();
+
+        let mut mask_generator = VisitedGenerator::new(TOTAL);
+        let k = 10;
+        let params = HnswQueryParams {
+            ef: 50,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+            use_acorn: false,
+        };
+        let query = fsl.value(0);
+        let truth: std::collections::HashSet<u32> =
+            brute_force_topk_masked(store.as_ref(), query.clone(), k, |id| id % 2 == 0)
+                .into_iter()
+                .collect();
+
+        let mut all_results = vec![];
+        for hnsw in [&builder, &loaded] {
+            let mut bitset = mask_generator.generate(TOTAL);
+            for id in (0..TOTAL as u32).step_by(2) {
+                bitset.insert(id);
+            }
+            let results = hnsw
+                .search_acorn(query.clone(), k, &params, &bitset, store.as_ref())
+                .unwrap();
+            assert_eq!(results.len(), k);
+            assert!(results.iter().all(|node| node.id % 2 == 0));
+            assert!(results.windows(2).all(|w| w[0].dist <= w[1].dist));
+            let hits = results.iter().filter(|n| truth.contains(&n.id)).count();
+            let recall = hits as f32 / k as f32;
+            assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})");
+            all_results.push(results);
+        }
+        assert_eq!(all_results[0], all_results[1]);
+
+        // default ef (k + k/2) and a deletion-style mask (all but a few rows)
+        let default_ef_params = HnswQueryParams {
+            ef: k + k / 2,
+            ..params
+        };
+        for excluded_stride in [2, 400] {
+            let passes = |id: u32| id % excluded_stride != 1;
+            let mut bitset = mask_generator.generate(TOTAL);
+            for id in (0..TOTAL as u32).filter(|id| passes(*id)) {
+                bitset.insert(id);
+            }
+            let truth: std::collections::HashSet<u32> =
+                brute_force_topk_masked(store.as_ref(), query.clone(), k, passes)
+                    .into_iter()
+                    .collect();
+            let results = builder
+                .search_acorn(
+                    query.clone(),
+                    k,
+                    &default_ef_params,
+                    &bitset,
+                    store.as_ref(),
+                )
+                .unwrap();
+            assert_eq!(results.len(), k);
+            assert!(results.iter().all(|node| passes(node.id)));
+            let hits = results.iter().filter(|n| truth.contains(&n.id)).count();
+            let recall = hits as f32 / k as f32;
+            assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})");
+        }
+    }
+
+    /// Dispatch: dense prefilters take the graph traversal, sparse ones the
+    /// exact flat scan, and both return only mask-passing row ids.
+    #[tokio::test]
+    async fn test_subindex_prefilter_dispatch() {
+        use arrow_array::cast::AsArray;
+        use async_trait::async_trait;
+        use lance_core::Result;
+        use lance_select::{RowAddrMask, RowAddrTreeMap};
+
+        use crate::metrics::NoOpMetricsCollector;
+        use crate::prefilter::PreFilter;
+
+        struct MaskPreFilter {
+            mask: Arc<RowAddrMask>,
+        }
+
+        #[async_trait]
+        impl PreFilter for MaskPreFilter {
+            async fn wait_for_ready(&self) -> Result<()> {
+                Ok(())
+            }
+            fn is_empty(&self) -> bool {
+                false
+            }
+            fn mask(&self) -> Arc<RowAddrMask> {
+                self.mask.clone()
+            }
+            fn filter_row_ids<'a>(
+                &self,
+                row_ids: Box<dyn Iterator<Item = &'a u64> + 'a>,
+            ) -> Vec<u64> {
+                self.mask.selected_indices(row_ids)
+            }
+        }
+
+        const DIM: usize = 32;
+        const TOTAL: usize = 2048;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+        let hnsw = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+
+        let k = 10;
+        let query_key = fsl.value(0);
+
+        let search_row_ids = |allowed: Vec<u64>, use_acorn: bool| {
+            let params = HnswQueryParams {
+                ef: 50,
+                lower_bound: None,
+                upper_bound: None,
+                dist_q_c: 0.0,
+                use_acorn,
+            };
+            let filter = Arc::new(MaskPreFilter {
+                mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(
+                    allowed,
+                ))),
+            });
+            let batch = hnsw
+                .search(
+                    query_key.clone(),
+                    k,
+                    params,
+                    store.as_ref(),
+                    filter,
+                    &NoOpMetricsCollector,
+                )
+                .unwrap();
+            batch[lance_core::ROW_ID]
+                .as_primitive::<arrow_array::types::UInt64Type>()
+                .values()
+                .to_vec()
+        };
+
+        // Dense mask (50% of rows), in both modes.
+        let dense: Vec<u64> = (0..TOTAL as u64).step_by(2).collect();
+        for use_acorn in [false, true] {
+            let row_ids = search_row_ids(dense.clone(), use_acorn);
+            assert_eq!(row_ids.len(), k);
+            assert!(row_ids.iter().all(|id| id % 2 == 0));
+        }
+
+        // All-pass mask: shortcuts to the unfiltered path.
+        let all: Vec<u64> = (0..TOTAL as u64).collect();
+        let unfiltered = hnsw
+            .search_basic(
+                query_key.clone(),
+                k,
+                &HnswQueryParams {
+                    ef: 50,
+                    lower_bound: None,
+                    upper_bound: None,
+                    dist_q_c: 0.0,
+                    use_acorn: false,
+                },
+                None,
+                store.as_ref(),
+            )
+            .unwrap();
+        let row_ids = search_row_ids(all, true);
+        assert_eq!(
+            row_ids,
+            unfiltered.iter().map(|n| n.id as u64).collect::<Vec<_>>()
+        );
+
+        // Sparse mask (< 10% of rows): the flat scan, which is exact.
+        let sparse: Vec<u64> = (0..TOTAL as u64).step_by(25).collect();
+        let row_ids = search_row_ids(sparse.clone(), true);
+        assert_eq!(row_ids.len(), k);
+        let truth = brute_force_topk_masked(store.as_ref(), query_key.clone(), k, |id| {
+            sparse.contains(&(id as u64))
+        });
+        let mut got: Vec<u32> = row_ids.iter().map(|id| *id as u32).collect();
+        got.sort_unstable();
+        let mut expected = truth;
+        expected.sort_unstable();
+        assert_eq!(got, expected);
+    }
+
     /// Regression guard for the `level_offsets` misalignment (issue #6746).
     /// `to_batch` writes the entry-point node at *every* level, but
     /// `level_count` only counts it at level 0, so the serialized batch has
@@ -1606,6 +1982,7 @@ mod tests {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let query = fsl.value(0);
         let builder_results = builder
@@ -1752,6 +2129,7 @@ mod tests {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let query = fsl.value(7);
         let a = builder
diff --git a/rust/lance-index/src/vector/hnsw/online.rs b/rust/lance-index/src/vector/hnsw/online.rs
index 8fbdcbcb1c5..83a471fd1ef 100644
--- a/rust/lance-index/src/vector/hnsw/online.rs
+++ b/rust/lance-index/src/vector/hnsw/online.rs
@@ -359,6 +359,7 @@ impl OnlineHnswBuilder {
                 lower_bound: None,
                 upper_bound: None,
                 dist_q_c: 0.0,
+                use_acorn: false,
             },
             dist_calc,
             None,
@@ -430,6 +431,7 @@ impl OnlineHnswBuilder {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let result = beam_search(
             &bottom,
@@ -662,6 +664,7 @@ mod tests {
                     lower_bound: None,
                     upper_bound: None,
                     dist_q_c: 0.0,
+                    use_acorn: false,
                 },
                 None,
                 &mut visited,
diff --git a/rust/lance-index/src/vector/kmeans.rs b/rust/lance-index/src/vector/kmeans.rs
index b11fb70bed0..ceff08c2d45 100644
--- a/rust/lance-index/src/vector/kmeans.rs
+++ b/rust/lance-index/src/vector/kmeans.rs
@@ -45,7 +45,7 @@ use {
 };
 
 use crate::vector::utils::SimpleIndex;
-use crate::{Error, Result};
+use lance_core::{Error, Result};
 
 /// KMean initialization method.
 #[derive(Debug, PartialEq)]
@@ -1156,7 +1156,16 @@ impl KMeans {
                 target_k
             );
         }
-        debug_assert_eq!(heap.len(), target_k);
+        if heap.len() < target_k {
+            return Err(ArrowError::InvalidArgumentError(format!(
+                "Cannot create {target_k} IVF partitions: k-means could only form {} non-empty \
+                 clusters from {n} training vectors. The dataset is likely too small or has too \
+                 many (near-)duplicate vectors for this many partitions. Reduce num_partitions to \
+                 <= {} or provide more diverse data.",
+                heap.len(),
+                heap.len()
+            )));
+        }
 
         // Construct final KMeans model with all centroids
         let mut all_clusters: Vec<Cluster<T::Native>> = heap.into_vec();
@@ -1722,6 +1731,47 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_hierarchical_kmeans_too_few_distinct_vectors_errors() {
+        // Regression test for https://github.com/lance-format/lance/issues/7867
+        //
+        // With a small number of distinct vectors repeated many times (heavy
+        // near-duplication) and dot distance, hierarchical k-means cannot form
+        // `target_k` non-empty clusters no matter how it splits: every split of a
+        // cluster of identical vectors is either ineffective (`all_same`) or
+        // immediately hits the "<= 1 point" floor. This used to trip
+        // `debug_assert_eq!(heap.len(), target_k)` (panic in debug builds) or
+        // silently return a half-empty centroid set (release builds). It should
+        // now return a clear error instead.
+        const DIM: usize = 8;
+        const NUM_DISTINCT: usize = 5;
+        const REPEATS: usize = 200;
+        const TARGET_K: usize = 300; // > 256 to trigger hierarchical clustering
+
+        let base_vectors = generate_random_array(NUM_DISTINCT * DIM);
+        let mut values = Vec::with_capacity(NUM_DISTINCT * REPEATS * DIM);
+        for _ in 0..REPEATS {
+            values.extend_from_slice(base_vectors.values());
+        }
+        let values = Float32Array::from(values);
+        let fsl = FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap();
+
+        let params = KMeansParams {
+            max_iters: 10,
+            hierarchical_k: 16,
+            distance_type: DistanceType::Dot,
+            ..Default::default()
+        };
+
+        let err = KMeans::new_with_params(&fsl, TARGET_K, &params)
+            .expect_err("training should fail rather than panic or silently under-produce");
+        let msg = err.to_string();
+        assert!(
+            msg.contains("Cannot create") && msg.contains(&TARGET_K.to_string()),
+            "unexpected error message: {msg}"
+        );
+    }
+
     #[tokio::test]
     async fn test_float16_underflow_fix() {
         // This test verifies the fix for float16 division underflow
diff --git a/rust/lance-index/src/vector/utils.rs b/rust/lance-index/src/vector/utils.rs
index fb4f9004c57..03fa2bb4b20 100644
--- a/rust/lance-index/src/vector/utils.rs
+++ b/rust/lance-index/src/vector/utils.rs
@@ -109,6 +109,7 @@ impl SimpleIndex {
             lower_bound: None,
             upper_bound: None,
             dist_q_c: 0.0,
+            use_acorn: false,
         };
         let res = match &self.store {
             SimpleStore::Float(store) => self.index.search_basic(query, 1, &params, None, store)?,
@@ -277,6 +278,7 @@ mod tests {
     use half::f16;
     use lance_arrow::FixedSizeListArrayExt;
     use num_traits::identities::Zero;
+    use rayon::ThreadPoolBuilder;
 
     use arrow::compute::cast;
     use rstest::rstest;
@@ -307,7 +309,8 @@ mod tests {
         (0..100).flat_map(|i| std::iter::repeat_n(i as f32, 16)).collect::<Vec<_>>(),
     )) as ArrayRef, 42.0f32)]
     fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef, #[case] query_val: f32) {
-        let index = build_index(centroids, 16);
+        let thread_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+        let index = thread_pool.install(|| build_index(centroids, 16));
         let query: ArrayRef = Arc::new(Float32Array::from(vec![query_val; 16]));
         let (id, dist) = index.search(query).unwrap();
         assert_eq!(id, 42);
diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml
index 6cee04d5fd9..643fef8c14a 100644
--- a/rust/lance-io/Cargo.toml
+++ b/rust/lance-io/Cargo.toml
@@ -35,8 +35,10 @@ byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
 futures.workspace = true
+goosefs-sdk = { workspace = true, optional = true }
 http.workspace = true
 log.workspace = true
+metrics = { workspace = true, optional = true }
 moka.workspace = true
 pin-project.workspace = true
 prost.workspace = true
@@ -60,6 +62,7 @@ rstest.workspace = true
 mock_instant.workspace = true
 tokio = { workspace = true, features = ["test-util"] }
 tracing-mock = { workspace = true }
+metrics-util = { workspace = true }
 
 [[bench]]
 name = "scheduler"
@@ -67,13 +70,15 @@ harness = false
 
 [features]
 default = ["aws", "azure", "gcp"]
+metrics = ["dep:metrics"]
 gcs-test = []
 goosefs-test = []
 gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"]
 aws = ["object_store/aws", "dep:aws-config", "dep:aws-credential-types", "dep:opendal", "opendal/services-s3", "dep:object_store_opendal"]
 azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "opendal/services-azdls", "dep:object_store_opendal"]
 oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"]
-goosefs = ["dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal"]
+# Pin goosefs-sdk until a release fixes the missing CString import in 0.1.6's Linux backend.
+goosefs = ["dep:goosefs-sdk", "dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal"]
 tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
 huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
 tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"]
diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs
index dafb5e5342f..15905ff4e69 100644
--- a/rust/lance-io/src/object_store.rs
+++ b/rust/lance-io/src/object_store.rs
@@ -40,6 +40,8 @@ pub(crate) mod dynamic_credentials;
 #[cfg(any(feature = "oss", feature = "huggingface", feature = "tos"))]
 pub(crate) mod dynamic_opendal;
 mod list_retry;
+#[cfg(feature = "metrics")]
+pub mod metrics;
 pub mod providers;
 pub mod storage_options;
 #[cfg(test)]
diff --git a/rust/lance-io/src/object_store/metrics.rs b/rust/lance-io/src/object_store/metrics.rs
new file mode 100644
index 00000000000..d11ae76bd22
--- /dev/null
+++ b/rust/lance-io/src/object_store/metrics.rs
@@ -0,0 +1,1602 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Publishes object store metrics via the [`metrics`] crate.
+//!
+//! Two layers cooperate:
+//!
+//! * [`MeteredObjectStore`] wraps any [`object_store::ObjectStore`] and records
+//!   per-operation request counts, transferred bytes, latency, errors, and the
+//!   number of requests currently in flight. It works for every store
+//!   regardless of backend.
+//! * [`MeteringHttpConnector`] wraps the HTTP client used by the native cloud
+//!   stores (S3 / GCS / Azure) and records throttle / retryable responses per
+//!   attempt. Because `object_store`'s retry loop re-issues each request
+//!   through the [`HttpService`](object_store::client::HttpService), this sees
+//!   every retried response, which a store-level wrapper cannot observe.
+//!
+//! The two layers have different coverage: every store gets the request-level
+//! metrics from [`MeteredObjectStore`], but only the native cloud stores get
+//! the HTTP-level throttle metrics. Opendal-backed stores (tos, oss, etc.)
+//! bypass `object_store`'s HTTP client, so there is no place to install the
+//! connector for them.
+//!
+//! Metrics carry a `base` label identifying the store. Its cardinality is
+//! controlled by the `LANCE_OBJECT_STORE_METRICS_LABEL` environment variable
+//! ([`BASE_LABEL_ENV_VAR`]):
+//!
+//! * `scheme` (default) — scheme only, e.g. `s3`; low, bounded cardinality.
+//! * `full` — the full store prefix, e.g. `s3$bucket` or `az$container@account`,
+//!   so multiple buckets on the same cloud can be told apart.
+//! * `off` — omit the `base` label entirely.
+//!
+//! The metric name constants ([`METRIC_REQUESTS`] etc.) and the recording
+//! helpers ([`record_request`], [`record_count`], [`record_error`],
+//! [`InFlightGuard`]) are public so custom object stores can emit the same
+//! metrics.
+
+use std::ops::Range;
+use std::pin::Pin;
+use std::sync::{Arc, OnceLock};
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+use bytes::Bytes;
+use futures::stream::BoxStream;
+use futures::{FutureExt, Stream, StreamExt};
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
+    UploadPart,
+};
+
+/// Total number of object store requests, labelled by `operation` and `base`.
+pub const METRIC_REQUESTS: &str = "lance_object_store_requests_total";
+/// Total bytes transferred by object store requests, labelled by `operation` and `base`.
+pub const METRIC_BYTES: &str = "lance_object_store_request_bytes_total";
+/// Object store request latency in seconds, labelled by `operation` and `base`.
+pub const METRIC_DURATION: &str = "lance_object_store_request_duration_seconds";
+/// Total number of failed object store requests, labelled by `operation` and `base`.
+pub const METRIC_ERRORS: &str = "lance_object_store_errors_total";
+/// Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer,
+/// labelled by `status` and `base`. Counts every attempt, including retries.
+pub const METRIC_THROTTLE: &str = "lance_object_store_throttle_total";
+/// Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP
+/// layer, labelled by `status` and `base`. Counts every attempt, including
+/// retries. This is a superset of [`METRIC_THROTTLE`]; 409 (conflict) is
+/// deliberately excluded so commit conflicts are not counted as retries.
+pub const METRIC_RETRYABLE: &str = "lance_object_store_retryable_responses_total";
+/// Number of object store requests currently in flight, labelled by `operation`
+/// and `base`.
+pub const METRIC_IN_FLIGHT: &str = "lance_object_store_in_flight_requests";
+
+/// Environment variable controlling the cardinality of the `base` label.
+pub const BASE_LABEL_ENV_VAR: &str = "LANCE_OBJECT_STORE_METRICS_LABEL";
+
+/// Controls how much of a store's identity the `base` label carries, traded off
+/// against metric cardinality. Selected via [`BASE_LABEL_ENV_VAR`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BaseLabelMode {
+    /// Full store prefix, e.g. `s3$bucket` or `az$container@account`. Highest
+    /// cardinality: one series family per bucket/container.
+    Full,
+    /// Scheme only, e.g. `s3`. The default: low, bounded cardinality.
+    Scheme,
+    /// Omit the `base` label entirely.
+    Off,
+}
+
+fn parse_base_label_mode(value: Option<&str>) -> BaseLabelMode {
+    match value {
+        Some("full") => BaseLabelMode::Full,
+        Some("off") | Some("none") => BaseLabelMode::Off,
+        Some("scheme") | None => BaseLabelMode::Scheme,
+        Some(other) => {
+            tracing::warn!(
+                "Unrecognized {BASE_LABEL_ENV_VAR}={other:?}; \
+                 expected one of full, scheme, off. Defaulting to scheme."
+            );
+            BaseLabelMode::Scheme
+        }
+    }
+}
+
+/// The label mode is read once from the environment and cached for the process.
+fn base_label_mode() -> BaseLabelMode {
+    static MODE: OnceLock<BaseLabelMode> = OnceLock::new();
+    *MODE.get_or_init(|| parse_base_label_mode(std::env::var(BASE_LABEL_ENV_VAR).ok().as_deref()))
+}
+
+/// Reduce a full store prefix (`scheme$authority`, or just `scheme` for stores
+/// without buckets) to the configured `base` label value, or `None` when the
+/// label should be omitted.
+fn scoped_base(mode: BaseLabelMode, base: &str) -> Option<String> {
+    match mode {
+        BaseLabelMode::Full => Some(base.to_owned()),
+        BaseLabelMode::Scheme => Some(base.split('$').next().unwrap_or(base).to_owned()),
+        BaseLabelMode::Off => None,
+    }
+}
+
+/// Build the `operation` (+ optional `base`) label set shared by all
+/// store-level metrics, honoring the configured label mode.
+fn operation_labels(base: &str, operation: &'static str) -> Vec<metrics::Label> {
+    let mut labels = vec![metrics::Label::new("operation", operation)];
+    if let Some(base) = scoped_base(base_label_mode(), base) {
+        labels.push(metrics::Label::new("base", base));
+    }
+    labels
+}
+
+/// Recommended histogram bucket boundaries for [`METRIC_DURATION`], in seconds.
+///
+/// Object store requests can take anywhere from a few milliseconds to the
+/// client timeout (commonly ~120s), so the boundaries are dense below 10s and
+/// keep useful resolution through the timeout band out to 5 minutes. Exporters
+/// that aggregate into fixed buckets (e.g. the OpenTelemetry bridge in the
+/// Python bindings) use these.
+pub const REQUEST_DURATION_BOUNDS: &[f64] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, // sub-10s
+    10.0, 20.0, 30.0, 45.0, 60.0, 90.0, 120.0, 150.0, 180.0, 240.0, 300.0, // 10s–5min
+];
+
+/// Register descriptions (units and help text) for the object store metrics.
+///
+/// This routes through whatever [`metrics::Recorder`] is currently installed,
+/// so it must be called *after* the recorder is set. Exporters that build a
+/// catalog of available metrics (such as the OpenTelemetry bridge) rely on
+/// these descriptions to discover metric names, kinds, and units up front.
+pub fn describe_metrics() {
+    metrics::describe_counter!(
+        METRIC_REQUESTS,
+        metrics::Unit::Count,
+        "Total number of object store requests, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_BYTES,
+        metrics::Unit::Bytes,
+        "Total bytes transferred by object store requests, by operation and scheme."
+    );
+    metrics::describe_histogram!(
+        METRIC_DURATION,
+        metrics::Unit::Seconds,
+        "Object store request latency in seconds, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_ERRORS,
+        metrics::Unit::Count,
+        "Total number of failed object store requests, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_THROTTLE,
+        metrics::Unit::Count,
+        "Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, by status and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_RETRYABLE,
+        metrics::Unit::Count,
+        "Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, by status and scheme."
+    );
+    metrics::describe_gauge!(
+        METRIC_IN_FLIGHT,
+        metrics::Unit::Count,
+        "Number of object store requests currently in flight, by operation and scheme."
+    );
+}
+
+/// Recommended fixed bucket boundaries for the histogram metrics defined here,
+/// as `(metric_name, boundaries)` pairs. Exporters that aggregate histograms
+/// into fixed buckets read this to configure each histogram.
+pub fn histogram_bounds() -> &'static [(&'static str, &'static [f64])] {
+    &[(METRIC_DURATION, REQUEST_DURATION_BOUNDS)]
+}
+
+/// Record the outcome of a unary request: count, latency, bytes (on success), and errors.
+pub fn record_request<T>(
+    base: &str,
+    operation: &'static str,
+    start: Instant,
+    bytes: u64,
+    result: &OSResult<T>,
+) {
+    record_outcome(base, operation, start, bytes, result.is_err());
+}
+
+/// Record count, latency, and either transferred bytes or an error for a
+/// completed request. Used both for unary requests and for streamed GETs whose
+/// bytes are only known once the body finishes.
+pub fn record_outcome(
+    base: &str,
+    operation: &'static str,
+    start: Instant,
+    bytes: u64,
+    is_error: bool,
+) {
+    let elapsed = start.elapsed().as_secs_f64();
+    let labels = operation_labels(base, operation);
+    metrics::counter!(METRIC_REQUESTS, labels.clone()).increment(1);
+    metrics::histogram!(METRIC_DURATION, labels.clone()).record(elapsed);
+    if is_error {
+        metrics::counter!(METRIC_ERRORS, labels).increment(1);
+    } else if bytes > 0 {
+        metrics::counter!(METRIC_BYTES, labels).increment(bytes);
+    }
+}
+
+/// Record a single request count without latency, used for streaming operations
+/// (list / delete) whose work happens lazily as the stream is polled.
+pub fn record_count(base: &str, operation: &'static str) {
+    metrics::counter!(METRIC_REQUESTS, operation_labels(base, operation)).increment(1);
+}
+
+/// Record a single error for an operation.
+pub fn record_error(base: &str, operation: &'static str) {
+    metrics::counter!(METRIC_ERRORS, operation_labels(base, operation)).increment(1);
+}
+
+/// Raises the in-flight gauge for an operation on creation and lowers it on
+/// drop, so the count stays balanced even if the request future or stream is
+/// cancelled or dropped before completing.
+pub struct InFlightGuard {
+    labels: Vec<metrics::Label>,
+}
+
+impl InFlightGuard {
+    pub fn new(base: &str, operation: &'static str) -> Self {
+        let labels = operation_labels(base, operation);
+        metrics::gauge!(METRIC_IN_FLIGHT, labels.clone()).increment(1.0);
+        Self { labels }
+    }
+}
+
+impl Drop for InFlightGuard {
+    fn drop(&mut self) {
+        metrics::gauge!(METRIC_IN_FLIGHT, self.labels.clone()).decrement(1.0);
+    }
+}
+
+#[derive(Debug)]
+pub struct MeteredObjectStore {
+    target: Arc<dyn object_store::ObjectStore>,
+    base: String,
+}
+
+impl std::fmt::Display for MeteredObjectStore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "MeteredObjectStore({})", self.target)
+    }
+}
+
+#[async_trait::async_trait]
+#[deny(clippy::missing_trait_methods)]
+impl object_store::ObjectStore for MeteredObjectStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        let size = bytes.content_length() as u64;
+        let _in_flight = InFlightGuard::new(&self.base, "put");
+        let start = Instant::now();
+        let result = self.target.put_opts(location, bytes, opts).await;
+        record_request(&self.base, "put", start, size, &result);
+        result
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        let upload = self.target.put_multipart_opts(location, opts).await?;
+        Ok(Box::new(MeteredMultipartUpload {
+            target: upload,
+            base: self.base.clone(),
+        }))
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        // `head()` is implemented as a `get_opts` call with `head = true`, so we
+        // distinguish it here to keep HEAD and GET as separate operations.
+        let is_head = options.head;
+        let operation = if is_head { "head" } else { "get" };
+        let in_flight = InFlightGuard::new(&self.base, operation);
+        let start = Instant::now();
+        let result = self.target.get_opts(location, options).await;
+
+        // A HEAD transfers only metadata, and errors carry no payload, so both
+        // are recorded immediately. `get_opts` only resolves once the response
+        // headers arrive; the body is streamed afterwards, so for a successful
+        // GET we defer recording until the body has been drained (see below).
+        if is_head || result.is_err() {
+            record_request(&self.base, operation, start, 0, &result);
+            return result;
+        }
+
+        let result = result.expect("checked to be Ok above");
+        Ok(meter_get_result(
+            result,
+            self.base.clone(),
+            start,
+            in_flight,
+        ))
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        let _in_flight = InFlightGuard::new(&self.base, "get");
+        let start = Instant::now();
+        let result = self.target.get_ranges(location, ranges).await;
+        let bytes = match &result {
+            Ok(parts) => parts.iter().map(|b| b.len() as u64).sum(),
+            Err(_) => 0,
+        };
+        record_request(&self.base, "get", start, bytes, &result);
+        result
+    }
+
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OSResult<Path>>,
+    ) -> BoxStream<'static, OSResult<Path>> {
+        let base = self.base.clone();
+        // Count one logical delete request per call, matching `list`: a single
+        // `delete_stream` maps to one batched request on stores that support it
+        // (e.g. S3's `DeleteObjects`), so counting per yielded path would
+        // over-count. Errors are still recorded per failing path.
+        record_count(&self.base, "delete");
+        let in_flight = InFlightGuard::new(&self.base, "delete");
+        self.target
+            .delete_stream(locations)
+            .map(move |result| {
+                // Reference `in_flight` so this `move` closure captures (owns)
+                // the guard, keeping the gauge raised until the stream is
+                // dropped (a move closure only captures the variables it uses).
+                let _in_flight = &in_flight;
+                if result.is_err() {
+                    record_error(&base, "delete");
+                }
+                result
+            })
+            .boxed()
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        record_count(&self.base, "list");
+        meter_list_stream(
+            self.target.list(prefix),
+            self.base.clone(),
+            InFlightGuard::new(&self.base, "list"),
+        )
+    }
+
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        record_count(&self.base, "list");
+        meter_list_stream(
+            self.target.list_with_offset(prefix, offset),
+            self.base.clone(),
+            InFlightGuard::new(&self.base, "list"),
+        )
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        let _in_flight = InFlightGuard::new(&self.base, "list");
+        let start = Instant::now();
+        let result = self.target.list_with_delimiter(prefix).await;
+        record_request(&self.base, "list", start, 0, &result);
+        result
+    }
+
+    async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "copy");
+        let start = Instant::now();
+        let result = self.target.copy_opts(from, to, opts).await;
+        record_request(&self.base, "copy", start, 0, &result);
+        result
+    }
+
+    async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "rename");
+        let start = Instant::now();
+        let result = self.target.rename_opts(from, to, opts).await;
+        record_request(&self.base, "rename", start, 0, &result);
+        result
+    }
+}
+
+/// Count errors yielded while draining a list stream. The request itself is
+/// counted once when the stream is created (a single LIST may return many items).
+fn meter_list_stream(
+    stream: BoxStream<'static, OSResult<ObjectMeta>>,
+    base: String,
+    in_flight: InFlightGuard,
+) -> BoxStream<'static, OSResult<ObjectMeta>> {
+    stream
+        .map(move |result| {
+            // Reference `in_flight` so this `move` closure captures (owns) the
+            // guard: a move closure only captures the variables it uses, and
+            // holding it here keeps the gauge raised until the stream is dropped.
+            let _in_flight = &in_flight;
+            if result.is_err() {
+                record_error(&base, "list");
+            }
+            result
+        })
+        .boxed()
+}
+
+/// Wrap a successful GET so the request is recorded once its body has been
+/// fully read, capturing the true transfer duration and byte count rather than
+/// the time-to-first-byte and declared range. For payloads without a body
+/// stream (e.g. a local file handle) the request is recorded immediately.
+fn meter_get_result(
+    mut result: GetResult,
+    base: String,
+    start: Instant,
+    in_flight: InFlightGuard,
+) -> GetResult {
+    match result.payload {
+        GetResultPayload::Stream(stream) => {
+            result.payload = GetResultPayload::Stream(
+                MeteredGetStream {
+                    inner: stream,
+                    base,
+                    start,
+                    bytes: 0,
+                    errored: false,
+                    recorded: false,
+                    _in_flight: in_flight,
+                }
+                .boxed(),
+            );
+            result
+        }
+        // No body stream to observe (e.g. a local file), so record now.
+        other => {
+            let bytes = result.range.end - result.range.start;
+            record_outcome(&base, "get", start, bytes, false);
+            result.payload = other;
+            result
+        }
+    }
+}
+
+/// Stream wrapper over a GET body that records the request (count, duration,
+/// bytes, errors) once the body is fully drained or the stream is dropped.
+struct MeteredGetStream {
+    inner: BoxStream<'static, OSResult<Bytes>>,
+    base: String,
+    start: Instant,
+    bytes: u64,
+    errored: bool,
+    recorded: bool,
+    _in_flight: InFlightGuard,
+}
+
+impl MeteredGetStream {
+    fn record(&mut self) {
+        if self.recorded {
+            return;
+        }
+        self.recorded = true;
+        record_outcome(&self.base, "get", self.start, self.bytes, self.errored);
+    }
+}
+
+impl Stream for MeteredGetStream {
+    type Item = OSResult<Bytes>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match self.inner.poll_next_unpin(cx) {
+            Poll::Ready(Some(Ok(chunk))) => {
+                self.bytes += chunk.len() as u64;
+                Poll::Ready(Some(Ok(chunk)))
+            }
+            Poll::Ready(Some(Err(e))) => {
+                self.errored = true;
+                Poll::Ready(Some(Err(e)))
+            }
+            Poll::Ready(None) => {
+                self.record();
+                Poll::Ready(None)
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl Drop for MeteredGetStream {
+    fn drop(&mut self) {
+        // Records the partial transfer if the body was dropped before it drained.
+        self.record();
+    }
+}
+
+#[derive(Debug)]
+struct MeteredMultipartUpload {
+    target: Box<dyn MultipartUpload>,
+    base: String,
+}
+
+#[async_trait::async_trait]
+impl MultipartUpload for MeteredMultipartUpload {
+    fn put_part(&mut self, data: PutPayload) -> UploadPart {
+        // Each part upload is a distinct request, recorded under the `put_part`
+        // operation with the same count / bytes / latency / error set as a
+        // unary put.
+        let base = self.base.clone();
+        let size = data.content_length() as u64;
+        let inner = self.target.put_part(data);
+        async move {
+            let _in_flight = InFlightGuard::new(&base, "put_part");
+            let start = Instant::now();
+            let result = inner.await;
+            record_request(&base, "put_part", start, size, &result);
+            result
+        }
+        .boxed()
+    }
+
+    async fn complete(&mut self) -> OSResult<PutResult> {
+        // Completing a multipart upload issues its own request that can throttle
+        // or fail, so it is metered like any other operation.
+        let _in_flight = InFlightGuard::new(&self.base, "complete_multipart");
+        let start = Instant::now();
+        let result = self.target.complete().await;
+        record_request(&self.base, "complete_multipart", start, 0, &result);
+        result
+    }
+
+    async fn abort(&mut self) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "abort_multipart");
+        let start = Instant::now();
+        let result = self.target.abort().await;
+        record_request(&self.base, "abort_multipart", start, 0, &result);
+        result
+    }
+}
+
+pub trait ObjectStoreMetricsExt {
+    /// Wrap this store so its operations publish metrics under the given `base` label.
+    fn metered(self, base: String) -> Arc<dyn object_store::ObjectStore>;
+}
+
+impl ObjectStoreMetricsExt for Arc<dyn object_store::ObjectStore> {
+    fn metered(self, base: String) -> Arc<dyn object_store::ObjectStore> {
+        Arc::new(MeteredObjectStore { target: self, base })
+    }
+}
+
+// --- Layer 2: HTTP-level throttle metrics for native cloud stores ---
+
+#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
+mod http {
+    use super::*;
+    use object_store::client::{
+        ClientOptions, HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse,
+        HttpService, ReqwestConnector,
+    };
+
+    /// An [`HttpConnector`] that records throttle and retryable responses
+    /// observed by the underlying HTTP client. Install it on the S3 / GCS /
+    /// Azure builders via `with_http_connector`.
+    #[derive(Debug)]
+    pub struct MeteringHttpConnector {
+        base: String,
+        inner: ReqwestConnector,
+    }
+
+    impl MeteringHttpConnector {
+        pub fn new(base: String) -> Self {
+            Self {
+                base,
+                inner: ReqwestConnector::default(),
+            }
+        }
+    }
+
+    impl HttpConnector for MeteringHttpConnector {
+        fn connect(&self, options: &ClientOptions) -> object_store::Result<HttpClient> {
+            let client = self.inner.connect(options)?;
+            Ok(HttpClient::new(MeteringHttpService {
+                base: self.base.clone(),
+                inner: client,
+            }))
+        }
+    }
+
+    #[derive(Debug)]
+    struct MeteringHttpService {
+        base: String,
+        inner: HttpClient,
+    }
+
+    #[async_trait::async_trait]
+    impl HttpService for MeteringHttpService {
+        async fn call(&self, req: HttpRequest) -> Result<HttpResponse, HttpError> {
+            let response = self.inner.execute(req).await?;
+            let status = response.status().as_u16();
+            // Each attempt that object_store may retry is recorded with its
+            // numeric status. Throttles (429 / 503) are a distinct, narrower
+            // signal than the broader set of retryable responses, so they get
+            // their own counter. 409 (conflict) is intentionally excluded from
+            // the retryable set so commit conflicts are not counted as retries.
+            let is_throttle = status == 429 || status == 503;
+            let is_retryable = status == 429 || status == 408 || (500..600).contains(&status);
+            if is_throttle {
+                metrics::counter!(METRIC_THROTTLE, status_labels(&self.base, status)).increment(1);
+            }
+            if is_retryable {
+                metrics::counter!(METRIC_RETRYABLE, status_labels(&self.base, status)).increment(1);
+            }
+            Ok(response)
+        }
+    }
+
+    /// Build the `status` (+ optional `base`) label set for HTTP-layer metrics,
+    /// honoring the configured label mode.
+    fn status_labels(base: &str, status: u16) -> Vec<metrics::Label> {
+        let mut labels = vec![metrics::Label::new("status", status.to_string())];
+        if let Some(base) = scoped_base(base_label_mode(), base) {
+            labels.push(metrics::Label::new("base", base));
+        }
+        labels
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+        use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+        use object_store::client::{HttpRequestBody, HttpResponseBody};
+
+        /// A mock [`HttpService`] that always responds with a fixed status code.
+        #[derive(Debug)]
+        struct StaticStatusService {
+            status: u16,
+        }
+
+        #[async_trait::async_trait]
+        impl HttpService for StaticStatusService {
+            async fn call(&self, _req: HttpRequest) -> Result<HttpResponse, HttpError> {
+                Ok(::http::Response::builder()
+                    .status(self.status)
+                    .body(HttpResponseBody::from(Bytes::new()))
+                    .unwrap())
+            }
+        }
+
+        fn request() -> HttpRequest {
+            ::http::Request::builder()
+                .method("GET")
+                .uri("http://example.com/obj")
+                .body(HttpRequestBody::empty())
+                .unwrap()
+        }
+
+        fn metric_count(
+            metrics: &[(metrics::Key, DebugValue)],
+            name: &str,
+            base: &str,
+            status: &str,
+        ) -> u64 {
+            for (key, value) in metrics {
+                if key.name() != name {
+                    continue;
+                }
+                let labels: std::collections::HashMap<&str, &str> =
+                    key.labels().map(|l| (l.key(), l.value())).collect();
+                if labels.get("base") == Some(&base)
+                    && labels.get("status") == Some(&status)
+                    && let DebugValue::Counter(v) = value
+                {
+                    return *v;
+                }
+            }
+            0
+        }
+
+        #[test]
+        fn test_throttle_and_retryable_responses_counted_by_status() {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                let rt = tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .unwrap();
+                rt.block_on(async {
+                    // Each attempt that object_store retries flows through `call`
+                    // again; here we simulate that by issuing several responses.
+                    // The base is baked into the connector, so it labels the
+                    // metric. Bases here have no `$`, so they are unaffected by
+                    // the label mode and this test isolates status handling.
+                    for (base, status) in [
+                        ("s3", 429u16),
+                        ("s3", 503),
+                        ("s3", 503),
+                        ("s3", 500),
+                        ("s3", 408),
+                        ("s3", 409),
+                        ("s3", 200),
+                        ("s3", 404),
+                        ("gs", 429),
+                    ] {
+                        let service = MeteringHttpService {
+                            base: base.into(),
+                            inner: HttpClient::new(StaticStatusService { status }),
+                        };
+                        service.call(request()).await.unwrap();
+                    }
+                });
+            });
+
+            let recorded: Vec<_> = snapshotter
+                .snapshot()
+                .into_vec()
+                .into_iter()
+                .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value))
+                .collect();
+
+            let throttle = |base, status| metric_count(&recorded, METRIC_THROTTLE, base, status);
+            let retryable = |base, status| metric_count(&recorded, METRIC_RETRYABLE, base, status);
+
+            // Throttles are only 429 and 503.
+            assert_eq!(throttle("s3", "429"), 1);
+            assert_eq!(throttle("s3", "503"), 2);
+            assert_eq!(throttle("s3", "500"), 0);
+            assert_eq!(throttle("s3", "408"), 0);
+
+            // Retryable is the broader set: 5xx, 429, 408 (but not 409).
+            assert_eq!(retryable("s3", "429"), 1);
+            assert_eq!(retryable("s3", "503"), 2);
+            assert_eq!(retryable("s3", "500"), 1);
+            assert_eq!(retryable("s3", "408"), 1);
+            // 409 conflict is excluded so commit conflicts are not counted as retries.
+            assert_eq!(retryable("s3", "409"), 0);
+
+            // Success and non-retryable client errors count as neither.
+            assert_eq!(throttle("s3", "200"), 0);
+            assert_eq!(retryable("s3", "404"), 0);
+
+            // The base label is taken from the connector, not shared across stores.
+            assert_eq!(throttle("gs", "429"), 1);
+            assert_eq!(retryable("gs", "429"), 1);
+            assert_eq!(throttle("gs", "503"), 0);
+        }
+    }
+}
+
+#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
+pub use http::MeteringHttpConnector;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
+    use object_store::memory::InMemory;
+    use object_store::{ObjectStoreExt, PutPayload};
+
+    fn payload(data: &[u8]) -> PutPayload {
+        PutPayload::from_bytes(Bytes::copy_from_slice(data))
+    }
+
+    fn metered_store() -> Arc<dyn object_store::ObjectStore> {
+        (Arc::new(InMemory::new()) as Arc<dyn object_store::ObjectStore>).metered("memory".into())
+    }
+
+    /// A single materialized snapshot of recorded metrics. It must be taken
+    /// only once: the snapshotter *drains* histogram samples on every
+    /// `snapshot()` call, so a second snapshot would see empty histograms.
+    type Metrics = Vec<(metrics::Key, DebugValue)>;
+
+    /// Materialize the current recorder state. Histogram samples are *drained*
+    /// on each call, so a metric must be read from a single snapshot.
+    fn snapshot(snapshotter: &Snapshotter) -> Metrics {
+        snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value))
+            .collect()
+    }
+
+    /// Run an async closure with a thread-local metrics recorder installed and
+    /// return the resulting metrics. Uses a current-thread runtime so all polls
+    /// happen on the thread that holds the recorder guard.
+    fn capture_metrics<F, Fut>(f: F) -> Metrics
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = ()>,
+    {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(f());
+        });
+        snapshot(&snapshotter)
+    }
+
+    fn key_matches(key: &metrics::Key, name: &str, labels: &[(&str, &str)]) -> bool {
+        if key.name() != name {
+            return false;
+        }
+        let actual: std::collections::HashSet<(&str, &str)> =
+            key.labels().map(|l| (l.key(), l.value())).collect();
+        labels.len() == actual.len() && labels.iter().all(|l| actual.contains(l))
+    }
+
+    fn counter_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> u64 {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Counter(v) = value
+            {
+                return *v;
+            }
+        }
+        0
+    }
+
+    fn histogram_count(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> usize {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Histogram(samples) = value
+            {
+                return samples.len();
+            }
+        }
+        0
+    }
+
+    fn gauge_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> f64 {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Gauge(v) = value
+            {
+                return v.0;
+            }
+        }
+        0.0
+    }
+
+    fn has_metric(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> bool {
+        metrics
+            .iter()
+            .any(|(key, _)| key_matches(key, name, labels))
+    }
+
+    #[test]
+    fn test_parse_base_label_mode() {
+        assert_eq!(parse_base_label_mode(None), BaseLabelMode::Scheme);
+        assert_eq!(parse_base_label_mode(Some("scheme")), BaseLabelMode::Scheme);
+        assert_eq!(parse_base_label_mode(Some("full")), BaseLabelMode::Full);
+        assert_eq!(parse_base_label_mode(Some("off")), BaseLabelMode::Off);
+        assert_eq!(parse_base_label_mode(Some("none")), BaseLabelMode::Off);
+        // Unrecognized values fall back to the conservative default.
+        assert_eq!(parse_base_label_mode(Some("bogus")), BaseLabelMode::Scheme);
+    }
+
+    #[test]
+    fn test_scoped_base() {
+        assert_eq!(
+            scoped_base(BaseLabelMode::Full, "s3$bucket").as_deref(),
+            Some("s3$bucket")
+        );
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "s3$bucket").as_deref(),
+            Some("s3")
+        );
+        // Azure keeps only the scheme even though its prefix carries the account.
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "az$container@account").as_deref(),
+            Some("az")
+        );
+        // A prefix without `$` (e.g. memory/file) is unchanged by scheme mode.
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "memory").as_deref(),
+            Some("memory")
+        );
+        assert_eq!(scoped_base(BaseLabelMode::Off, "s3$bucket"), None);
+    }
+
+    #[test]
+    fn test_base_label_defaults_to_scheme() {
+        // No env var is set in the test process, so the default `scheme` mode
+        // applies: the full prefix collapses to just the scheme.
+        let recorded = capture_metrics(|| async {
+            let store = (Arc::new(InMemory::new()) as Arc<dyn object_store::ObjectStore>)
+                .metered("s3$my-bucket".into());
+            store.put(&Path::from("a"), payload(b"x")).await.unwrap();
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "put"), ("base", "s3")]
+            ),
+            1
+        );
+        // The full prefix is not emitted as the label under the default mode.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "put"), ("base", "s3$my-bucket")]
+            ),
+            0
+        );
+    }
+
+    #[test]
+    fn test_put_records_count_bytes_and_latency() {
+        let data = b"hello world";
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store
+                .put(&Path::from("a/b.bin"), payload(data))
+                .await
+                .unwrap();
+        });
+
+        let labels = [("operation", "put"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(
+            counter_value(&recorded, METRIC_BYTES, &labels),
+            data.len() as u64
+        );
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_get_records_count_and_bytes() {
+        let data = b"hello world";
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(data)).await.unwrap();
+            // The GET is only recorded once its body has been fully drained.
+            store.get(&path).await.unwrap().bytes().await.unwrap();
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(
+            counter_value(&recorded, METRIC_BYTES, &labels),
+            data.len() as u64
+        );
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_get_not_recorded_until_body_drained() {
+        let data = b"hello world";
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                let path = Path::from("a/b.bin");
+                store.put(&path, payload(data)).await.unwrap();
+
+                // Holding the result without reading the body records nothing yet.
+                let result = store.get(&path).await.unwrap();
+                assert_eq!(
+                    counter_value(&snapshot(&snapshotter), METRIC_REQUESTS, &labels),
+                    0
+                );
+
+                // Draining the body records the request with the true byte count.
+                let bytes = result.bytes().await.unwrap();
+                assert_eq!(bytes.len(), data.len());
+                let recorded = snapshot(&snapshotter);
+                assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+                assert_eq!(
+                    counter_value(&recorded, METRIC_BYTES, &labels),
+                    data.len() as u64
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_head_is_a_separate_operation() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello world")).await.unwrap();
+            store.head(&path).await.unwrap();
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "head"), ("base", "memory")]
+            ),
+            1
+        );
+        // The head call must not be counted as a get.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "get"), ("base", "memory")]
+            ),
+            0
+        );
+        // A HEAD transfers only metadata, so it records no payload bytes.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_BYTES,
+                &[("operation", "head"), ("base", "memory")]
+            ),
+            0
+        );
+    }
+
+    #[test]
+    fn test_delete_records_one_request_per_call() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}.bin")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            // `delete` drives `delete_stream`; deleting three paths is still one
+            // logical delete request (a single batched request on real stores).
+            let paths =
+                futures::stream::iter((0..3).map(|i| Ok(Path::from(format!("a/{i}.bin"))))).boxed();
+            let _: Vec<_> = store.delete_stream(paths).collect().await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "delete"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_list_counts_one_request_not_per_item() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}.bin")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            let _: Vec<_> = store.list(Some(&Path::from("a"))).collect().await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "list"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_error_is_counted() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            // Getting a missing object errors.
+            let _ = store.get(&Path::from("does/not/exist")).await;
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1);
+        // A failed request is still counted as a request, with latency recorded.
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        // No bytes are transferred on a failed get.
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_get_ranges_sums_part_bytes_and_labels_get() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello world")).await.unwrap();
+            // Two disjoint ranges of 3 bytes each.
+            store.get_ranges(&path, &[2..5, 6..9]).await.unwrap();
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 6);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_copy_and_rename_record_zero_bytes() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store
+                .put(&Path::from("a/src"), payload(b"x"))
+                .await
+                .unwrap();
+            store
+                .copy(&Path::from("a/src"), &Path::from("a/copy"))
+                .await
+                .unwrap();
+            store
+                .rename(&Path::from("a/copy"), &Path::from("a/moved"))
+                .await
+                .unwrap();
+        });
+
+        for operation in ["copy", "rename"] {
+            let labels = [("operation", operation), ("base", "memory")];
+            assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+            assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+            assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        }
+    }
+
+    #[test]
+    fn test_list_with_delimiter_records_latency() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store.put(&Path::from("a/b"), payload(b"x")).await.unwrap();
+            store
+                .list_with_delimiter(Some(&Path::from("a")))
+                .await
+                .unwrap();
+        });
+
+        let labels = [("operation", "list"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_list_with_offset_counts_one_request() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            let _: Vec<_> = store
+                .list_with_offset(Some(&Path::from("a")), &Path::from("a/0"))
+                .collect()
+                .await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "list"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_multipart_records_each_part_and_complete() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            upload.put_part(payload(b"hello")).await.unwrap(); // 5 bytes
+            upload.put_part(payload(b"world!!")).await.unwrap(); // 7 bytes
+            upload.complete().await.unwrap();
+        });
+
+        let part_labels = [("operation", "put_part"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &part_labels), 2);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &part_labels), 12);
+        // Each part records its own latency sample, like a unary put.
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &part_labels), 2);
+        // A successful part upload records no error.
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &part_labels), 0);
+
+        // Completing the upload is its own metered request.
+        let complete_labels = [("operation", "complete_multipart"), ("base", "memory")];
+        assert_eq!(
+            counter_value(&recorded, METRIC_REQUESTS, &complete_labels),
+            1
+        );
+        assert_eq!(
+            histogram_count(&recorded, METRIC_DURATION, &complete_labels),
+            1
+        );
+    }
+
+    #[test]
+    fn test_multipart_abort_is_recorded() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            upload.put_part(payload(b"hello")).await.unwrap();
+            upload.abort().await.unwrap();
+        });
+
+        let labels = [("operation", "abort_multipart"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_multipart_part_error_is_counted() {
+        let recorded = capture_metrics(|| async {
+            let store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            let _ = upload.put_part(payload(b"data")).await;
+        });
+
+        let labels = [("operation", "put_part"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        // A failed part transfers no counted bytes.
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_in_flight_guard_tracks_and_releases() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let g1 = InFlightGuard::new("memory", "get");
+            let g2 = InFlightGuard::new("memory", "get");
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                2.0
+            );
+            drop(g1);
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                1.0
+            );
+            drop(g2);
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                0.0
+            );
+        });
+    }
+
+    #[test]
+    fn test_in_flight_gauge_is_wired_and_balances() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello")).await.unwrap();
+            store.get(&path).await.unwrap();
+        });
+
+        // The gauge is emitted for each operation (guard is wired in) and, once
+        // the operation completes, balances back to zero.
+        for operation in ["put", "get"] {
+            let labels = [("operation", operation), ("base", "memory")];
+            assert!(has_metric(&recorded, METRIC_IN_FLIGHT, &labels));
+            assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0);
+        }
+    }
+
+    #[test]
+    fn test_list_stream_holds_in_flight_until_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "list"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                store.put(&Path::from("a/x"), payload(b"x")).await.unwrap();
+
+                // Creating the stream raises the gauge; it stays raised until the
+                // stream is dropped, even before any items are drained.
+                let stream = store.list(Some(&Path::from("a")));
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(stream);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_delete_stream_holds_in_flight_until_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "delete"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                let locations = futures::stream::iter(vec![Ok(Path::from("a/b"))]).boxed();
+
+                // Like list, creating the delete stream raises the gauge and holds
+                // it until the stream is dropped, before any items are drained.
+                let stream = store.delete_stream(locations);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(stream);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_in_flight_released_when_operation_future_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let started = Arc::new(tokio::sync::Notify::new());
+                // Never signalled: the request stays blocked mid-flight.
+                let release = Arc::new(tokio::sync::Notify::new());
+                let store = (Arc::new(BlockingStore {
+                    started: started.clone(),
+                    release,
+                }) as Arc<dyn object_store::ObjectStore>)
+                    .metered("memory".into());
+
+                let path = Path::from("a/b");
+                let mut fut = Box::pin(store.get(&path));
+                // Drive the request until it is blocked inside the inner store.
+                tokio::select! {
+                    _ = &mut fut => unreachable!("the blocking store never returns"),
+                    _ = started.notified() => {}
+                }
+
+                // The gauge is raised while the request is outstanding, and
+                // dropping the future before it completes releases it.
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(fut);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_streaming_errors_are_counted() {
+        let recorded = capture_metrics(|| async {
+            let delete_store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let _ = delete_store.delete(&Path::from("a/b")).await;
+
+            let list_store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let _: Vec<_> = list_store.list(None).collect().await;
+        });
+
+        // delete_stream counts the item and records an error when it fails.
+        let delete_labels = [("operation", "delete"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &delete_labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &delete_labels), 1);
+
+        // A list request is counted once; a failure while draining records an error.
+        let list_labels = [("operation", "list"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &list_labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &list_labels), 1);
+    }
+
+    /// A store whose stream-producing operations always yield an error, used to
+    /// exercise the error branches of the streaming wrappers.
+    #[derive(Debug)]
+    struct FailingStreamStore;
+
+    impl std::fmt::Display for FailingStreamStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "FailingStreamStore")
+        }
+    }
+
+    fn test_error() -> object_store::Error {
+        object_store::Error::Generic {
+            store: "FailingStreamStore",
+            source: "injected failure".into(),
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl object_store::ObjectStore for FailingStreamStore {
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            Ok(Box::new(FailingUpload))
+        }
+
+        async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult<GetResult> {
+            unimplemented!()
+        }
+
+        fn delete_stream(
+            &self,
+            _locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            futures::stream::once(async { Err(test_error()) }).boxed()
+        }
+
+        fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            futures::stream::once(async { Err(test_error()) }).boxed()
+        }
+
+        fn list_with_offset(
+            &self,
+            _prefix: Option<&Path>,
+            _offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult<ListResult> {
+            unimplemented!()
+        }
+
+        async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> {
+            unimplemented!()
+        }
+
+        async fn rename_opts(
+            &self,
+            _from: &Path,
+            _to: &Path,
+            _opts: RenameOptions,
+        ) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+
+    /// A [`MultipartUpload`] whose part uploads always fail, used to exercise the
+    /// error branch of the metered `put_part`.
+    #[derive(Debug)]
+    struct FailingUpload;
+
+    #[async_trait::async_trait]
+    impl MultipartUpload for FailingUpload {
+        fn put_part(&mut self, _data: PutPayload) -> UploadPart {
+            async { Err(test_error()) }.boxed()
+        }
+
+        async fn complete(&mut self) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn abort(&mut self) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+
+    /// A store whose `get_opts` blocks after signalling `started`, so a request
+    /// can be observed mid-flight and then dropped before it completes.
+    #[derive(Debug)]
+    struct BlockingStore {
+        started: Arc<tokio::sync::Notify>,
+        release: Arc<tokio::sync::Notify>,
+    }
+
+    impl std::fmt::Display for BlockingStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "BlockingStore")
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl object_store::ObjectStore for BlockingStore {
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            unimplemented!()
+        }
+
+        async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult<GetResult> {
+            self.started.notify_one();
+            self.release.notified().await;
+            unreachable!("release is never signalled in the test")
+        }
+
+        fn delete_stream(
+            &self,
+            _locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            unimplemented!()
+        }
+
+        fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        fn list_with_offset(
+            &self,
+            _prefix: Option<&Path>,
+            _offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult<ListResult> {
+            unimplemented!()
+        }
+
+        async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> {
+            unimplemented!()
+        }
+
+        async fn rename_opts(
+            &self,
+            _from: &Path,
+            _to: &Path,
+            _opts: RenameOptions,
+        ) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+}
diff --git a/rust/lance-io/src/object_store/providers.rs b/rust/lance-io/src/object_store/providers.rs
index 775c98552a8..5e7963e5e97 100644
--- a/rust/lance-io/src/object_store/providers.rs
+++ b/rust/lance-io/src/object_store/providers.rs
@@ -265,6 +265,14 @@ impl ObjectStoreRegistry {
 
         store.inner = store.inner.traced();
 
+        #[cfg(feature = "metrics")]
+        {
+            // Label metrics by the store's unique prefix (e.g. `s3$bucket`,
+            // `az$container@account`) so multiple stores on one cloud differ.
+            use crate::object_store::metrics::ObjectStoreMetricsExt;
+            store.inner = store.inner.metered(cache_path.clone());
+        }
+
         if let Some(wrapper) = &params.object_store_wrapper {
             store.inner = wrapper.wrap(&cache_path, store.inner);
         }
diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs
index 9aad637bce2..8cad196f087 100644
--- a/rust/lance-io/src/object_store/providers/aws.rs
+++ b/rust/lance-io/src/object_store/providers/aws.rs
@@ -73,6 +73,12 @@ impl AwsStoreProvider {
             s3_storage_options.insert(AmazonS3ConfigKey::S3Express, true.to_string());
         }
 
+        // Compute the metrics label before rewriting the url below, so it
+        // matches the prefix the registry uses to key this store.
+        #[cfg(feature = "metrics")]
+        let store_prefix =
+            self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?;
+
         // before creating the OSObjectStore we need to rewrite the url to drop ddb related parts
         base_path.set_scheme("s3").unwrap();
         base_path.set_query(None);
@@ -89,6 +95,13 @@ impl AwsStoreProvider {
             .with_retry(retry_config)
             .with_region(region);
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(store_prefix),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 
diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs
index e61f3f3b364..9de866bff91 100644
--- a/rust/lance-io/src/object_store/providers/azure.rs
+++ b/rust/lance-io/src/object_store/providers/azure.rs
@@ -192,6 +192,15 @@ impl AzureBlobStoreProvider {
             builder = builder.with_credentials(credentials);
         }
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(
+                    self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?,
+                ),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 
diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs
index f7f0a7672ff..35f79b66804 100644
--- a/rust/lance-io/src/object_store/providers/gcp.rs
+++ b/rust/lance-io/src/object_store/providers/gcp.rs
@@ -89,6 +89,15 @@ impl GcsStoreProvider {
             builder = builder.with_credentials(credential_provider);
         }
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(
+                    self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?,
+                ),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 }
diff --git a/rust/lance-io/src/object_writer.rs b/rust/lance-io/src/object_writer.rs
index 0fd0a30f9e7..6dd6c4d992f 100644
--- a/rust/lance-io/src/object_writer.rs
+++ b/rust/lance-io/src/object_writer.rs
@@ -392,25 +392,23 @@ impl AsyncWrite for ObjectWriter {
                     let fut = Box::pin(async move { store.put_multipart(path.as_ref()).await });
                     self.state = UploadState::CreatingUpload(fut);
                 }
+                // TODO: Make max concurrency configurable from storage options.
                 UploadState::InProgress {
                     upload,
                     part_idx,
                     futures,
                     ..
-                } => {
-                    // TODO: Make max concurrency configurable from storage options.
-                    if futures.len() < max_upload_parallelism() {
-                        let data = Self::next_part_buffer(
-                            &mut mut_self.buffer,
-                            *part_idx,
-                            mut_self.use_constant_size_upload_parts,
-                        );
-                        futures.spawn(
-                            Self::put_part(upload.as_mut(), data, *part_idx, None)
-                                .instrument(tracing::Span::current()),
-                        );
-                        *part_idx += 1;
-                    }
+                } if futures.len() < max_upload_parallelism() => {
+                    let data = Self::next_part_buffer(
+                        &mut mut_self.buffer,
+                        *part_idx,
+                        mut_self.use_constant_size_upload_parts,
+                    );
+                    futures.spawn(
+                        Self::put_part(upload.as_mut(), data, *part_idx, None)
+                            .instrument(tracing::Span::current()),
+                    );
+                    *part_idx += 1;
                 }
                 _ => {}
             }
diff --git a/rust/lance-io/src/scheduler.rs b/rust/lance-io/src/scheduler.rs
index 0c2d4dc44bd..e24587d7559 100644
--- a/rust/lance-io/src/scheduler.rs
+++ b/rust/lance-io/src/scheduler.rs
@@ -29,6 +29,7 @@ mod lite;
 const BACKPRESSURE_MIN: u64 = 5;
 // Don't log backpressure warnings more than once / minute
 const BACKPRESSURE_DEBOUNCE: u64 = 60;
+const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state";
 
 // Global counter of how many IOPS we have issued
 static IOPS_COUNTER: AtomicU64 = AtomicU64::new(0);
@@ -83,11 +84,23 @@ impl PrioritiesInFlight {
             self.in_flight.remove(pos);
         }
     }
+
+    fn len(&self) -> usize {
+        self.in_flight.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.in_flight.is_empty()
+    }
 }
 
 struct IoQueueState {
+    // The configured number of IOPS that can be issued concurrently.
+    io_capacity: u32,
     // Number of IOPS we can issue concurrently before pausing I/O
     iops_avail: u32,
+    // The configured byte budget for unread I/O.
+    io_buffer_size: u64,
     // Number of bytes we are allowed to buffer in memory before pausing I/O
     //
     // This can dip below 0 due to I/O prioritization
@@ -110,7 +123,9 @@ struct IoQueueState {
 impl IoQueueState {
     fn new(io_capacity: u32, io_buffer_size: u64) -> Self {
         Self {
+            io_capacity,
             iops_avail: io_capacity,
+            io_buffer_size,
             bytes_avail: io_buffer_size as i64,
             pending_requests: BinaryHeap::new(),
             priorities_in_flight: PrioritiesInFlight::new(io_capacity),
@@ -121,6 +136,65 @@ impl IoQueueState {
         }
     }
 
+    fn scheduler_state_event(&self) -> Option<SchedulerStateEvent> {
+        if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) {
+            return None;
+        }
+
+        let pending_bytes = self
+            .pending_requests
+            .iter()
+            .map(IoTask::num_bytes)
+            .sum::<u64>();
+        let head_task = self.pending_requests.peek();
+        let min_in_flight_priority = if self.priorities_in_flight.is_empty() {
+            None
+        } else {
+            Some(self.priorities_in_flight.min_in_flight())
+        };
+        let head_task_priority_bypass = head_task.map(|task| {
+            self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight()
+        });
+        let head_task_blocked_by_iops = head_task.map(|_| self.iops_avail == 0);
+        let head_task_blocked_by_bytes = head_task.map(|task| {
+            let bypasses_bytes = self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight();
+            !bypasses_bytes && task.num_bytes() as i64 > self.bytes_avail
+        });
+        let head_task_can_deliver = head_task.map(|task| self.can_deliver_without_warning(task));
+        let head_task_bytes = head_task.map(IoTask::num_bytes);
+        let (head_task_priority_high, head_task_priority_low) =
+            split_priority(head_task.map(|task| task.priority));
+        let (min_in_flight_priority_high, min_in_flight_priority_low) =
+            split_priority(min_in_flight_priority);
+
+        Some(SchedulerStateEvent {
+            queue_kind: "standard",
+            io_capacity: u64::from(self.io_capacity),
+            iops_available: u64::from(self.iops_avail),
+            active_iops: u64::from(self.io_capacity.saturating_sub(self.iops_avail)),
+            pending_iops: self.pending_requests.len() as u64,
+            pending_bytes,
+            bytes_available: self.bytes_avail,
+            bytes_reserved: self.io_buffer_size as i64 - self.bytes_avail,
+            io_buffer_size_bytes: self.io_buffer_size,
+            priorities_in_flight: self.priorities_in_flight.len() as u64,
+            no_backpressure: self.no_backpressure,
+            head_task_bytes,
+            head_task_priority_high,
+            head_task_priority_low,
+            min_in_flight_priority_high,
+            min_in_flight_priority_low,
+            head_task_can_deliver,
+            head_task_priority_bypass,
+            head_task_blocked_by_iops,
+            head_task_blocked_by_bytes,
+        })
+    }
+
     fn warn_if_needed(&self) {
         let seconds_elapsed = self.start.elapsed().as_secs();
         let last_warn = self.last_warn.load(Ordering::Acquire);
@@ -140,6 +214,20 @@ impl IoQueueState {
     }
 
     fn can_deliver(&self, task: &IoTask) -> bool {
+        let can_deliver = self.can_deliver_without_warning(task);
+        if !can_deliver
+            && self.iops_avail > 0
+            && !(self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight())
+            && task.num_bytes() as i64 > self.bytes_avail
+        {
+            self.warn_if_needed();
+        }
+        can_deliver
+    }
+
+    fn can_deliver_without_warning(&self, task: &IoTask) -> bool {
         if self.iops_avail == 0 {
             false
         } else if self.no_backpressure
@@ -151,11 +239,8 @@ impl IoQueueState {
             || self.priorities_in_flight.contains(task.priority)
         {
             true
-        } else if task.num_bytes() as i64 > self.bytes_avail {
-            self.warn_if_needed();
-            false
         } else {
-            true
+            task.num_bytes() as i64 <= self.bytes_avail
         }
     }
 
@@ -191,13 +276,15 @@ struct IoQueue {
     state: Mutex<IoQueueState>,
     // Used to signal new I/O requests have arrived that might potentially be runnable
     notify: Notify,
+    stats: IoStats,
 }
 
 impl IoQueue {
-    fn new(io_capacity: u32, io_buffer_size: u64) -> Self {
+    fn new(io_capacity: u32, io_buffer_size: u64, stats: IoStats) -> Self {
         Self {
             state: Mutex::new(IoQueueState::new(io_capacity, io_buffer_size)),
             notify: Notify::new(),
+            stats,
         }
     }
 
@@ -208,9 +295,12 @@ impl IoQueue {
             task.priority >> 64,
             task.priority & 0xFFFFFFFFFFFFFFFF
         );
-        let mut state = self.state.lock().unwrap();
-        state.pending_requests.push(task);
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.pending_requests.push(task);
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
@@ -220,6 +310,9 @@ impl IoQueue {
             {
                 let mut state = self.state.lock().unwrap();
                 if let Some(task) = state.next_task() {
+                    let event = state.scheduler_state_event();
+                    drop(state);
+                    emit_scheduler_state_event(event, &self.stats);
                     return Some(task);
                 }
 
@@ -233,29 +326,39 @@ impl IoQueue {
     }
 
     fn on_iop_complete(&self) {
-        let mut state = self.state.lock().unwrap();
-        state.iops_avail += 1;
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.iops_avail += 1;
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
 
     fn on_bytes_consumed(&self, bytes: u64, priority: u128, num_reqs: usize) {
-        let mut state = self.state.lock().unwrap();
-        state.bytes_avail += bytes as i64;
-        for _ in 0..num_reqs {
-            state.priorities_in_flight.remove(priority);
-        }
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.bytes_avail += bytes as i64;
+            for _ in 0..num_reqs {
+                state.priorities_in_flight.remove(priority);
+            }
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
 
     fn close(&self) {
-        let mut state = self.state.lock().unwrap();
-        state.done_scheduling = true;
-        let pending_requests = std::mem::take(&mut state.pending_requests);
-        drop(state);
+        let (pending_requests, event) = {
+            let mut state = self.state.lock().unwrap();
+            state.done_scheduling = true;
+            let pending_requests = std::mem::take(&mut state.pending_requests);
+            let event = state.scheduler_state_event();
+            (pending_requests, event)
+        };
+        emit_scheduler_state_event(event, &self.stats);
         for request in pending_requests {
             request.cancel();
         }
@@ -274,9 +377,13 @@ struct MutableBatch<F: FnOnce(Response) + Send> {
     num_bytes: u64,
     priority: u128,
     num_reqs: usize,
+    num_delivered: usize,
     err: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
     // When true, report 0 bytes consumed so the backpressure budget is unaffected
     bypass_backpressure: bool,
+    // Queue the batch's backpressure reservation is refunded to once its response
+    // is delivered or discarded (see `Response`'s `Drop`).
+    io_queue: Arc<IoQueue>,
 }
 
 impl<F: FnOnce(Response) + Send> MutableBatch<F> {
@@ -286,6 +393,7 @@ impl<F: FnOnce(Response) + Send> MutableBatch<F> {
         priority: u128,
         num_reqs: usize,
         bypass_backpressure: bool,
+        io_queue: Arc<IoQueue>,
     ) -> Self {
         Self {
             when_done: Some(when_done),
@@ -293,8 +401,10 @@ impl<F: FnOnce(Response) + Send> MutableBatch<F> {
             num_bytes: 0,
             priority,
             num_reqs,
+            num_delivered: 0,
             err: None,
             bypass_backpressure,
+            io_queue,
         }
     }
 }
@@ -305,9 +415,16 @@ impl<F: FnOnce(Response) + Send> MutableBatch<F> {
 // data.
 impl<F: FnOnce(Response) + Send> Drop for MutableBatch<F> {
     fn drop(&mut self) {
-        // If we have an error, return that.  Otherwise return the data
-        let result = if self.err.is_some() {
-            Err(Error::wrapped(self.err.take().unwrap()))
+        // If we have an error, return that. Otherwise return the data, as long as the I/O requests have been processed.
+        let result = if let Some(err) = self.err.take() {
+            Err(Error::wrapped(err))
+        } else if self.num_delivered < self.data_buffers.len() {
+            // This usually happens on tokio runtime shutdown
+            Err(Error::io(format!(
+                "I/O request was dropped before completion ({} of {} reads delivered)",
+                self.num_delivered,
+                self.data_buffers.len()
+            )))
         } else {
             let mut data = Vec::new();
             std::mem::swap(&mut data, &mut self.data_buffers);
@@ -316,7 +433,8 @@ impl<F: FnOnce(Response) + Send> Drop for MutableBatch<F> {
         // We don't really care if no one is around to receive it, just let
         // the result go out of scope and get cleaned up
         let response = Response {
-            data: result,
+            data: Some(result),
+            io_queue: self.io_queue.clone(),
             // Report 0 bytes for bypass tasks so the backpressure budget is unaffected
             num_bytes: if self.bypass_backpressure {
                 0
@@ -344,6 +462,7 @@ impl<F: FnOnce(Response) + Send> DataSink for MutableBatch<F> {
     // Called by worker tasks to add data to the MutableBatch
     fn deliver_data(&mut self, data: DataChunk) {
         self.num_bytes += data.num_bytes;
+        self.num_delivered += 1;
         match data.data {
             Ok(data_bytes) => {
                 self.data_buffers[data.task_idx] = data_bytes;
@@ -519,6 +638,84 @@ impl ScanStats {
     }
 }
 
+fn split_priority(priority: Option<u128>) -> (Option<u64>, Option<u64>) {
+    priority
+        .map(|priority| ((priority >> 64) as u64, priority as u64))
+        .unzip()
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(super) struct SchedulerStateEvent {
+    pub(super) queue_kind: &'static str,
+    pub(super) io_capacity: u64,
+    pub(super) iops_available: u64,
+    pub(super) active_iops: u64,
+    pub(super) pending_iops: u64,
+    pub(super) pending_bytes: u64,
+    pub(super) bytes_available: i64,
+    pub(super) bytes_reserved: i64,
+    pub(super) io_buffer_size_bytes: u64,
+    pub(super) priorities_in_flight: u64,
+    pub(super) no_backpressure: bool,
+    pub(super) head_task_bytes: Option<u64>,
+    pub(super) head_task_priority_high: Option<u64>,
+    pub(super) head_task_priority_low: Option<u64>,
+    pub(super) min_in_flight_priority_high: Option<u64>,
+    pub(super) min_in_flight_priority_low: Option<u64>,
+    pub(super) head_task_can_deliver: Option<bool>,
+    pub(super) head_task_priority_bypass: Option<bool>,
+    pub(super) head_task_blocked_by_iops: Option<bool>,
+    pub(super) head_task_blocked_by_bytes: Option<bool>,
+}
+
+impl SchedulerStateEvent {
+    fn trace(self, stats: ScanStats) {
+        tracing::event!(
+            target: SCHEDULER_STATE_EVENT_TARGET,
+            tracing::Level::TRACE,
+            queue_kind = self.queue_kind,
+            scheduler_iops = stats.iops,
+            scheduler_requests = stats.requests,
+            scheduler_bytes_read = stats.bytes_read,
+            io_capacity = self.io_capacity,
+            iops_available = self.iops_available,
+            active_iops = self.active_iops,
+            pending_iops = self.pending_iops,
+            pending_bytes = self.pending_bytes,
+            bytes_available = self.bytes_available,
+            bytes_reserved = self.bytes_reserved,
+            io_buffer_size_bytes = self.io_buffer_size_bytes,
+            priorities_in_flight = self.priorities_in_flight,
+            no_backpressure = self.no_backpressure,
+            head_task_bytes_present = self.head_task_bytes.is_some(),
+            head_task_bytes = self.head_task_bytes.unwrap_or_default(),
+            head_task_priority_high_present = self.head_task_priority_high.is_some(),
+            head_task_priority_high = self.head_task_priority_high.unwrap_or_default(),
+            head_task_priority_low_present = self.head_task_priority_low.is_some(),
+            head_task_priority_low = self.head_task_priority_low.unwrap_or_default(),
+            min_in_flight_priority_high_present = self.min_in_flight_priority_high.is_some(),
+            min_in_flight_priority_high = self.min_in_flight_priority_high.unwrap_or_default(),
+            min_in_flight_priority_low_present = self.min_in_flight_priority_low.is_some(),
+            min_in_flight_priority_low = self.min_in_flight_priority_low.unwrap_or_default(),
+            head_task_can_deliver_present = self.head_task_can_deliver.is_some(),
+            head_task_can_deliver = self.head_task_can_deliver.unwrap_or(false),
+            head_task_priority_bypass_present = self.head_task_priority_bypass.is_some(),
+            head_task_priority_bypass = self.head_task_priority_bypass.unwrap_or(false),
+            head_task_blocked_by_iops_present = self.head_task_blocked_by_iops.is_some(),
+            head_task_blocked_by_iops = self.head_task_blocked_by_iops.unwrap_or(false),
+            head_task_blocked_by_bytes_present = self.head_task_blocked_by_bytes.is_some(),
+            head_task_blocked_by_bytes = self.head_task_blocked_by_bytes.unwrap_or(false),
+            "Scheduler state"
+        );
+    }
+}
+
+pub(super) fn emit_scheduler_state_event(event: Option<SchedulerStateEvent>, stats: &IoStats) {
+    if let Some(event) = event {
+        event.trace(stats.snapshot());
+    }
+}
+
 /// A shareable, cloneable handle to a set of cumulative I/O counters.
 ///
 /// All clones share the same underlying counters.  This serves two purposes:
@@ -598,12 +795,25 @@ impl Debug for ScanScheduler {
 }
 
 struct Response {
-    data: Result<Vec<Bytes>>,
+    // `Option` so the caller can take the data out while the response (and its
+    // backpressure refund on drop) stays intact.
+    data: Option<Result<Vec<Bytes>>>,
+    io_queue: Arc<IoQueue>,
     priority: u128,
     num_reqs: usize,
     num_bytes: u64,
 }
 
+// Refund the batch's backpressure reservation when the response is dropped, be
+// that on delivery or when a cancelled request's undelivered response is
+// discarded.  This releases the budget even if the caller drops the future early.
+impl Drop for Response {
+    fn drop(&mut self) {
+        self.io_queue
+            .on_bytes_consumed(self.num_bytes, self.priority, self.num_reqs);
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct SchedulerConfig {
     /// the # of bytes that can be buffered but not yet requested.
@@ -659,6 +869,7 @@ impl ScanScheduler {
     /// * config - configuration settings for the scheduler
     pub fn new(object_store: Arc<ObjectStore>, config: SchedulerConfig) -> Arc<Self> {
         let io_capacity = object_store.io_parallelism();
+        let stats = IoStats::new();
         let use_lite = config
             .use_lite_scheduler
             .unwrap_or_else(|| object_store.prefers_lite_scheduler());
@@ -666,12 +877,14 @@ impl ScanScheduler {
             let io_queue = Arc::new(lite::IoQueue::new(
                 io_capacity as u64,
                 config.io_buffer_size_bytes,
+                stats.clone(),
             ));
             IoQueueType::Lite(io_queue)
         } else {
             let io_queue = Arc::new(IoQueue::new(
                 io_capacity as u32,
                 config.io_buffer_size_bytes,
+                stats.clone(),
             ));
             let io_queue_clone = io_queue.clone();
             // Best we can do here is fire and forget.  If the I/O loop is still running when the scheduler is
@@ -683,7 +896,7 @@ impl ScanScheduler {
         Arc::new(Self {
             object_store,
             io_queue,
-            stats: IoStats::new(),
+            stats,
         })
     }
 
@@ -782,6 +995,7 @@ impl ScanScheduler {
             priority,
             request.len(),
             bypass_backpressure,
+            io_queue.clone(),
         ))));
 
         for (task_idx, iop) in request.into_iter().enumerate() {
@@ -820,14 +1034,11 @@ impl ScanScheduler {
 
         self.do_submit_request(reader, request, tx, priority, io_queue, bypass_backpressure);
 
-        let io_queue_clone = io_queue.clone();
-
-        rx.map(move |wrapped_rsp| {
-            // Right now, it isn't possible for I/O to be cancelled so a cancel error should
-            // not occur
-            let rsp = wrapped_rsp.unwrap();
-            io_queue_clone.on_bytes_consumed(rsp.num_bytes, rsp.priority, rsp.num_reqs);
-            rsp.data
+        rx.map(|wrapped_rsp| {
+            // A cancel error can't occur: the sender always sends before dropping.
+            // The reservation is refunded on `Response` drop, so just take the data.
+            let mut rsp = wrapped_rsp.unwrap();
+            rsp.data.take().unwrap()
         })
     }
 
@@ -1159,6 +1370,47 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_scheduler_state_event_fields() {
+        use tracing_mock::{expect, subscriber};
+
+        let event = expect::event()
+            .with_target(SCHEDULER_STATE_EVENT_TARGET)
+            .at_level(tracing::Level::TRACE)
+            .with_fields(
+                expect::field("queue_kind")
+                    .with_value(&"standard")
+                    .and(expect::field("scheduler_iops").with_value(&7u64))
+                    .and(expect::field("scheduler_requests").with_value(&3u64))
+                    .and(expect::field("scheduler_bytes_read").with_value(&4096u64))
+                    .and(expect::field("io_capacity").with_value(&4u64))
+                    .and(expect::field("pending_iops").with_value(&1u64))
+                    .and(expect::field("bytes_available").with_value(&128i64))
+                    .and(expect::field("head_task_bytes_present").with_value(&true))
+                    .and(expect::field("head_task_bytes").with_value(&1u64))
+                    .and(expect::field("head_task_can_deliver_present").with_value(&true))
+                    .and(expect::field("head_task_can_deliver").with_value(&true)),
+            );
+        let (subscriber, handle) = subscriber::mock().event(event).run_with_handle();
+
+        let stats = IoStats::new();
+        stats.add_scan_stats(&ScanStats {
+            iops: 7,
+            requests: 3,
+            bytes_read: 4096,
+        });
+        let mut state = IoQueueState::new(4, 192);
+        state.iops_avail = 2;
+        state.bytes_avail = 128;
+        state.pending_requests.push(make_task(1, false));
+
+        tracing::subscriber::with_default(subscriber, || {
+            emit_scheduler_state_event(state.scheduler_state_event(), &stats);
+        });
+
+        handle.assert_finished();
+    }
+
     #[test]
     fn test_iotask_ordering() {
         // Bypass tasks must come out of the heap before non-bypass tasks.
@@ -1176,6 +1428,29 @@ mod tests {
         assert_eq!(order, vec![(5, true), (20, true), (1, false), (10, false)]);
     }
 
+    #[test]
+    fn test_batch_with_undelivered_slot_is_error() {
+        let response = Arc::new(Mutex::new(None));
+        let response_clone = response.clone();
+        let io_queue = Arc::new(IoQueue::new(1, 1024, IoStats::new()));
+        let batch = MutableBatch::new(
+            move |rsp| *response_clone.lock().unwrap() = Some(rsp),
+            2, // num_data_buffers
+            0, // priority
+            2, // num_reqs
+            false,
+            io_queue,
+        );
+        drop(batch);
+
+        let mut rsp = response.lock().unwrap().take().unwrap();
+        let data = rsp.data.take().unwrap();
+        assert!(
+            data.is_err(),
+            "undelivered slot must yield an error, got {data:?}",
+        );
+    }
+
     #[tokio::test]
     async fn test_full_seq_read() {
         let tmp_file = TempObjFile::default();
@@ -1474,6 +1749,136 @@ mod tests {
         assert!(second_fut.await.unwrap().unwrap().len() == 20);
     }
 
+    #[tokio::test]
+    async fn test_standard_scheduler_state_tracks_queue_state() {
+        let some_path = Path::parse("foo").unwrap();
+        let base_store = Arc::new(InMemory::new());
+        base_store
+            .put(&some_path, vec![0; 1000].into())
+            .await
+            .unwrap();
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+        let mut obj_store = MockObjectStore::default();
+        let semaphore_copy = semaphore.clone();
+        obj_store
+            .expect_get_opts()
+            .returning(move |location, options| {
+                let semaphore = semaphore.clone();
+                let base_store = base_store.clone();
+                let location = location.clone();
+                async move {
+                    semaphore.acquire().await.unwrap().forget();
+                    base_store.get_opts(&location, options).await
+                }
+                .boxed()
+            });
+        let obj_store = Arc::new(ObjectStore::new(
+            Arc::new(obj_store),
+            Url::parse("mem://").unwrap(),
+            Some(500),
+            None,
+            false,
+            false,
+            1,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+
+        let scheduler = ScanScheduler::new(
+            obj_store,
+            SchedulerConfig {
+                io_buffer_size_bytes: 1024 * 1024,
+                use_lite_scheduler: Some(false),
+            },
+        );
+        let file_scheduler = scheduler
+            .open_file(&Path::parse("foo").unwrap(), &CachedFileSize::new(1000))
+            .await
+            .unwrap();
+
+        let first_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..10, 0),
+        )
+        .boxed();
+        let second_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..20, 100),
+        )
+        .boxed();
+        let third_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..30, 0),
+        )
+        .boxed();
+
+        let io_queue = match &scheduler.io_queue {
+            IoQueueType::Standard(io_queue) => io_queue.clone(),
+            IoQueueType::Lite(_) => unreachable!("test forces the standard scheduler"),
+        };
+        let (
+            io_capacity,
+            iops_available,
+            pending_bytes,
+            bytes_reserved,
+            priorities_in_flight,
+            head_task_bytes,
+            head_task_blocked_by_iops,
+            head_task_blocked_by_bytes,
+        ) = timeout(Duration::from_secs(5), async {
+            loop {
+                let observed = {
+                    let state = io_queue.state.lock().unwrap();
+                    let active_iops = state.io_capacity.saturating_sub(state.iops_avail);
+                    if active_iops == 1 && state.pending_requests.len() == 2 {
+                        let pending_bytes = state
+                            .pending_requests
+                            .iter()
+                            .map(IoTask::num_bytes)
+                            .sum::<u64>();
+                        let head_task = state.pending_requests.peek().unwrap();
+                        let bypasses_bytes = state.no_backpressure
+                            || head_task.bypass_backpressure
+                            || head_task.priority <= state.priorities_in_flight.min_in_flight();
+                        Some((
+                            state.io_capacity,
+                            state.iops_avail,
+                            pending_bytes,
+                            state.io_buffer_size as i64 - state.bytes_avail,
+                            state.priorities_in_flight.len(),
+                            head_task.num_bytes(),
+                            state.iops_avail == 0,
+                            !bypasses_bytes && head_task.num_bytes() as i64 > state.bytes_avail,
+                        ))
+                    } else {
+                        None
+                    }
+                };
+                if let Some(observed) = observed {
+                    break observed;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .unwrap();
+
+        assert_eq!(io_capacity, 1);
+        assert_eq!(iops_available, 0);
+        assert_eq!(pending_bytes, 50);
+        assert_eq!(bytes_reserved, 10);
+        assert_eq!(priorities_in_flight, 1);
+        assert_eq!(head_task_bytes, 30);
+        assert!(head_task_blocked_by_iops);
+        assert!(!head_task_blocked_by_bytes);
+
+        semaphore_copy.add_permits(3);
+        assert_eq!(first_fut.await.unwrap().unwrap().len(), 10);
+        assert_eq!(third_fut.await.unwrap().unwrap().len(), 30);
+        assert_eq!(second_fut.await.unwrap().unwrap().len(), 20);
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     async fn test_backpressure() {
         let some_path = Path::parse("foo").unwrap();
@@ -1609,6 +2014,7 @@ mod tests {
     #[derive(Debug)]
     struct BlockingReader {
         semaphore: Arc<tokio::sync::Semaphore>,
+        get_range_count: Arc<AtomicU64>,
         path: Path,
     }
 
@@ -1639,6 +2045,7 @@ mod tests {
             &self,
             range: Range<usize>,
         ) -> futures::future::BoxFuture<'static, object_store::Result<Bytes>> {
+            self.get_range_count.fetch_add(1, Ordering::Release);
             let semaphore = self.semaphore.clone();
             let num_bytes = range.end - range.start;
             Box::pin(async move {
@@ -1675,6 +2082,7 @@ mod tests {
         let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
         let reader: Arc<dyn Reader> = Arc::new(BlockingReader {
             semaphore: semaphore.clone(),
+            get_range_count: Arc::new(AtomicU64::new(0)),
             path: Path::parse("test").unwrap(),
         });
 
@@ -1981,4 +2389,105 @@ mod tests {
             .unwrap();
         assert_eq!(bytes_dispatched.load(Ordering::Acquire), 30);
     }
+
+    // Against a 100-byte budget: submit fut1 (50 bytes, priority 0), drop it while
+    // its read is still blocked in get_range, then submit fut2 (60 bytes, priority 1).
+    // fut2's priority can't win the priority-bypass, so it needs 60 of the budget --
+    // available only if fut1's dropped reservation was refunded. Returns whether fut2
+    // completed within 2s (false = the reservation leaked and fut2 deadlocked).
+    async fn run_caller_drop_scenario(use_lite_scheduler: bool) -> (bool, Duration) {
+        let obj_store = Arc::new(ObjectStore::new(
+            Arc::new(InMemory::new()),
+            Url::parse("mem://").unwrap(),
+            Some(4096),
+            None,
+            false,
+            false,
+            1,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        let scheduler = ScanScheduler::new(
+            obj_store,
+            SchedulerConfig {
+                io_buffer_size_bytes: 100,
+                use_lite_scheduler: Some(use_lite_scheduler),
+            },
+        );
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+        let get_range_count = Arc::new(AtomicU64::new(0));
+        let reader: Arc<dyn Reader> = Arc::new(BlockingReader {
+            semaphore: semaphore.clone(),
+            get_range_count: get_range_count.clone(),
+            path: Path::parse("test").unwrap(),
+        });
+
+        // Step 1: reserve 50 of the 100 budget bytes with a read we never consume.
+        // Spawn it so we can cancel the caller-side future while it is still parked
+        // waiting for the (blocked) read to finish.
+        let fut1 = scheduler.submit_request(reader.clone(), vec![0..50], 0, false);
+        let handle = tokio::spawn(async move {
+            let _ = fut1.await;
+        });
+
+        // Wait until the read is genuinely in flight (blocked on the semaphore).
+        // This guarantees the 50-byte reservation has been taken before we drop
+        // the caller, closing the race between the I/O loop and the abort.
+        while get_range_count.load(Ordering::Acquire) == 0 {
+            tokio::time::sleep(Duration::from_millis(1)).await;
+        }
+
+        // Step 2: drop the caller-side future while its `rx` is still pending.
+        handle.abort();
+        let _ = handle.await;
+
+        // Step 3: let the in-flight read finish. The reservation should be refunded
+        // now that the request is done, whether or not the caller is still around.
+        semaphore.add_permits(1);
+        // Give the read time to run to completion so the refund would already have
+        // happened.
+        tokio::time::sleep(Duration::from_millis(50)).await;
+
+        // Step 4: submit the follow-up. Add a permit up front so that, if it *is*
+        // admitted, its own read can complete rather than block on the semaphore.
+        semaphore.add_permits(1);
+        let fut2 = scheduler.submit_request(reader, vec![100..160], 1, false);
+
+        let start = std::time::Instant::now();
+        let outcome = timeout(Duration::from_secs(2), fut2).await;
+        let elapsed = start.elapsed();
+        match outcome {
+            Ok(res) => {
+                assert_eq!(res.unwrap().iter().map(|b| b.len()).sum::<usize>(), 60);
+                (true, elapsed)
+            }
+            Err(_) => (false, elapsed),
+        }
+    }
+
+    /// Dropping a standard-scheduler request future while its read is in flight must
+    /// still refund the backpressure reservation, so a later request that needs the
+    /// budget does not deadlock.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn standard_scheduler_refunds_reservation_on_caller_drop() {
+        let (completed, elapsed) = run_caller_drop_scenario(false).await;
+        assert!(
+            completed,
+            "standard scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \
+             the dropped request's reservation was not refunded"
+        );
+    }
+
+    /// Same guarantee for the lite scheduler: dropping a request future mid-read
+    /// releases its reservation via the `TaskHandle` drop path.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn lite_scheduler_refunds_reservation_on_caller_drop() {
+        let (completed, elapsed) = run_caller_drop_scenario(true).await;
+        assert!(
+            completed,
+            "lite scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \
+             the dropped request's reservation was not refunded"
+        );
+    }
 }
diff --git a/rust/lance-io/src/scheduler/lite.rs b/rust/lance-io/src/scheduler/lite.rs
index fc666139a05..b7a06e11bb7 100644
--- a/rust/lance-io/src/scheduler/lite.rs
+++ b/rust/lance-io/src/scheduler/lite.rs
@@ -33,7 +33,10 @@ use std::{
 use bytes::Bytes;
 use lance_core::{Error, Result};
 
-use super::{BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN};
+use super::{
+    BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN, IoStats, SCHEDULER_STATE_EVENT_TARGET,
+    SchedulerStateEvent, emit_scheduler_state_event,
+};
 
 type RunFn = Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send>> + Send>;
 
@@ -67,6 +70,26 @@ enum TaskState {
     },
 }
 
+impl TaskState {
+    fn backpressure_reservation(&self) -> Option<BackpressureReservation> {
+        match self {
+            Self::Reserved {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Running {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Finished {
+                backpressure_reservation,
+                ..
+            } => Some(*backpressure_reservation),
+            Self::Initial { .. } | Self::Broken => None,
+        }
+    }
+}
+
 /// A custom error type that might have a backpressure reservation
 ///
 /// This is used instead of Lance's standard error type so we can ensure
@@ -85,25 +108,14 @@ impl BrokenTaskError {
     // This will capture any backpressure reservation the task has and put it into the
     // error so we make sure to release it when returning the error.
     fn new(task_state: TaskState, message: String) -> Self {
-        match task_state {
-            TaskState::Reserved {
-                backpressure_reservation,
-                ..
-            }
-            | TaskState::Running {
-                backpressure_reservation,
-                ..
-            }
-            | TaskState::Finished {
-                backpressure_reservation,
-                ..
-            } => Self {
+        match task_state.backpressure_reservation() {
+            None => Self {
                 message,
-                backpressure_reservation: Some(backpressure_reservation),
+                backpressure_reservation: None,
             },
-            TaskState::Broken | TaskState::Initial { .. } => Self {
+            Some(reservation) => Self {
                 message,
-                backpressure_reservation: None,
+                backpressure_reservation: Some(reservation),
             },
         }
     }
@@ -237,6 +249,7 @@ trait BackpressureThrottle: Send {
     /// Unconditionally acquire a zero-cost reservation, tracking only the priority.
     /// Used for bypass tasks that must never be blocked by backpressure.
     fn force_acquire(&mut self, priority: u128) -> BackpressureReservation;
+    fn state(&self) -> BackpressureState;
 }
 
 // We want to allow requests that have a lower priority than any
@@ -279,9 +292,22 @@ impl PrioritiesInFlight {
             self.in_flight.remove(pos);
         }
     }
+
+    fn len(&self) -> usize {
+        self.in_flight.len()
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BackpressureState {
+    max_bytes: u64,
+    bytes_available: i64,
+    priorities_in_flight: u64,
+    no_backpressure: bool,
 }
 
 struct SimpleBackpressureThrottle {
+    max_bytes: u64,
     start: Instant,
     last_warn: AtomicU64,
     bytes_available: i64,
@@ -297,6 +323,7 @@ impl SimpleBackpressureThrottle {
             panic!("Max bytes must be less than {}", i64::MAX);
         }
         Self {
+            max_bytes,
             start: Instant::now(),
             last_warn: AtomicU64::new(0),
             bytes_available: max_bytes as i64,
@@ -358,6 +385,15 @@ impl BackpressureThrottle for SimpleBackpressureThrottle {
             priority,
         }
     }
+
+    fn state(&self) -> BackpressureState {
+        BackpressureState {
+            max_bytes: self.max_bytes,
+            bytes_available: self.bytes_available,
+            priorities_in_flight: self.priorities_in_flight.len() as u64,
+            no_backpressure: self.no_backpressure,
+        }
+    }
 }
 
 struct TaskEntry {
@@ -427,6 +463,48 @@ impl IoQueueState {
             Ok(())
         }
     }
+
+    fn scheduler_state_event(&self) -> Option<SchedulerStateEvent> {
+        if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) {
+            return None;
+        }
+
+        let backpressure = self.backpressure_throttle.state();
+        let pending_bytes = self
+            .pending_tasks
+            .iter()
+            .filter_map(|entry| self.tasks.get(&entry.task_id))
+            .map(|task| task.num_bytes)
+            .sum::<u64>();
+        let active_iops = self
+            .tasks
+            .values()
+            .filter(|task| matches!(task.state, TaskState::Running { .. }))
+            .count() as u64;
+
+        Some(SchedulerStateEvent {
+            queue_kind: "lite",
+            io_capacity: 0,
+            iops_available: 0,
+            active_iops,
+            pending_iops: self.pending_tasks.len() as u64,
+            pending_bytes,
+            bytes_available: backpressure.bytes_available,
+            bytes_reserved: backpressure.max_bytes as i64 - backpressure.bytes_available,
+            io_buffer_size_bytes: backpressure.max_bytes,
+            priorities_in_flight: backpressure.priorities_in_flight,
+            no_backpressure: backpressure.no_backpressure,
+            head_task_bytes: None,
+            head_task_priority_high: None,
+            head_task_priority_low: None,
+            min_in_flight_priority_high: None,
+            min_in_flight_priority_low: None,
+            head_task_can_deliver: None,
+            head_task_priority_bypass: None,
+            head_task_blocked_by_iops: None,
+            head_task_blocked_by_bytes: None,
+        })
+    }
 }
 
 /// A queue of I/O tasks to be shared between the I/O scheduler and the I/O decoder.
@@ -449,12 +527,14 @@ impl IoQueueState {
 /// day as well)
 pub(super) struct IoQueue {
     state: Arc<Mutex<IoQueueState>>,
+    stats: IoStats,
 }
 
 impl IoQueue {
-    pub fn new(max_concurrency: u64, max_bytes: u64) -> Self {
+    pub fn new(max_concurrency: u64, max_bytes: u64, stats: IoStats) -> Self {
         Self {
             state: Arc::new(Mutex::new(IoQueueState::new(max_concurrency, max_bytes))),
+            stats,
         }
     }
 
@@ -471,6 +551,9 @@ impl IoQueue {
             state.handle_result(task.reserve(reservation))?;
             state.handle_result(task.start())?;
             state.tasks.insert(task_id, task);
+            let event = state.scheduler_state_event();
+            drop(state);
+            emit_scheduler_state_event(event, &self.stats);
             return Ok(());
         }
 
@@ -480,6 +563,9 @@ impl IoQueue {
             reserved: task.is_reserved(),
         });
         state.tasks.insert(task_id, task);
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
         Ok(())
     }
 
@@ -518,34 +604,42 @@ impl IoQueue {
 
     // When a task completes we should check to see if any other tasks are now runnable
     fn on_task_complete(&self, mut state: MutexGuard<IoQueueState>) -> Result<()> {
-        let state_ref = &mut *state;
-        let mut task_result = TaskResult::Ok(());
-        while !state_ref.pending_tasks.is_empty() {
-            // Unwrap safe here since we just checked the queue is not empty
-            let next_task = state_ref.pending_tasks.peek().unwrap();
-            let Some(task) = state_ref.tasks.get_mut(&next_task.task_id) else {
-                log::warn!("Task with id {} was lost", next_task.task_id);
-                continue;
-            };
-            if !task.is_reserved() {
-                let Some(reservation) = state_ref
-                    .backpressure_throttle
-                    .try_acquire(task.num_bytes, task.priority)
-                else {
-                    break;
+        let result = {
+            let state_ref = &mut *state;
+            let mut task_result = TaskResult::Ok(());
+            while !state_ref.pending_tasks.is_empty() {
+                // Unwrap safe here since we just checked the queue is not empty
+                let task_id = state_ref.pending_tasks.peek().unwrap().task_id;
+                let Some(task) = state_ref.tasks.get_mut(&task_id) else {
+                    // The caller dropped this task's handle (see `abandon`); discard the
+                    // stale queue entry instead of spinning on it.
+                    state_ref.pending_tasks.pop();
+                    continue;
                 };
-                if let Err(e) = task.reserve(reservation) {
+                if !task.is_reserved() {
+                    let Some(reservation) = state_ref
+                        .backpressure_throttle
+                        .try_acquire(task.num_bytes, task.priority)
+                    else {
+                        break;
+                    };
+                    if let Err(e) = task.reserve(reservation) {
+                        task_result = Err(e);
+                        break;
+                    }
+                }
+                state_ref.pending_tasks.pop();
+                if let Err(e) = task.start() {
                     task_result = Err(e);
                     break;
                 }
             }
-            state_ref.pending_tasks.pop();
-            if let Err(e) = task.start() {
-                task_result = Err(e);
-                break;
-            }
-        }
-        state_ref.handle_result(task_result)
+            state_ref.handle_result(task_result)
+        };
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
+        result
     }
 
     fn poll(&self, task_id: u64, cx: &mut Context<'_>) -> Poll<Result<Bytes>> {
@@ -573,10 +667,34 @@ impl IoQueue {
     }
 
     pub(super) fn close(&self) {
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            for task in std::mem::take(&mut state.tasks).values_mut() {
+                task.cancel();
+            }
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
+    }
+
+    // Called when a caller drops a task's handle before the task finishes.  Removes
+    // the task and returns any backpressure reservation it holds to the budget, then
+    // re-checks the queue so newly-affordable tasks can start.  Unlike the standard
+    // release path (`poll`), this runs without the task being polled to completion,
+    // so a cancelled read does not leak its reservation.
+    fn abandon(&self, task_id: u64) {
         let mut state = self.state.lock().unwrap();
-        for task in std::mem::take(&mut state.tasks).values_mut() {
-            task.cancel();
+        let Some(task) = state.tasks.remove(&task_id) else {
+            // Already consumed by `poll`; nothing to release.
+            return;
+        };
+
+        if let Some(reservation) = task.state.backpressure_reservation() {
+            state.backpressure_throttle.release(reservation);
         }
+        // Freed budget may make queued tasks runnable; there is no caller to surface
+        // an error to here.
+        let _ = self.on_task_complete(state);
     }
 }
 
@@ -592,6 +710,12 @@ impl Future for TaskHandle {
     }
 }
 
+impl Drop for TaskHandle {
+    fn drop(&mut self) {
+        self.queue.abandon(self.task_id);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -600,7 +724,7 @@ mod tests {
     #[tokio::test]
     async fn test_priority_ordering() {
         // Backpressure budget of 10 bytes: only one 10-byte task runs at a time.
-        let queue = Arc::new(IoQueue::new(128, 10));
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
 
         // Records the priority of each task when its run_fn is invoked (i.e. when
         // the task transitions to Running).
@@ -708,7 +832,7 @@ mod tests {
     async fn test_zero_buffer_bypasses_backpressure() {
         // Budget = 0 sets no_backpressure = true, so all tasks start immediately
         // regardless of how many bytes are "outstanding".
-        let queue = Arc::new(IoQueue::new(128, 0));
+        let queue = Arc::new(IoQueue::new(128, 0, IoStats::default()));
         let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
 
         let make_run_fn =
@@ -750,7 +874,7 @@ mod tests {
     async fn test_bypass_flag_proceeds_past_exhausted_budget() {
         // Budget of 10 bytes. A blocker task fills it. A task with bypass=true starts
         // immediately despite the exhausted budget; a normal task stays queued.
-        let queue = Arc::new(IoQueue::new(128, 10));
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
         let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
 
         let make_run_fn =
diff --git a/rust/lance-linalg/Cargo.toml b/rust/lance-linalg/Cargo.toml
index 6a188ec3c62..d94efa7b2fa 100644
--- a/rust/lance-linalg/Cargo.toml
+++ b/rust/lance-linalg/Cargo.toml
@@ -25,6 +25,7 @@ approx = { workspace = true }
 criterion = { workspace = true }
 lance-testing = { path = "../lance-testing" }
 proptest.workspace = true
+rstest.workspace = true
 
 [build-dependencies]
 cc = "1.0.83"
diff --git a/rust/lance-linalg/proptest-regressions/distance/cosine.txt b/rust/lance-linalg/proptest-regressions/distance/cosine.txt
new file mode 100644
index 00000000000..04e6dcb967f
--- /dev/null
+++ b/rust/lance-linalg/proptest-regressions/distance/cosine.txt
@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 679c764b02e2726ec2d943b3abdc7d7d3565e168d4081b155860c1e69b616fce # shrinks to (x, y) = ([-1.2933521e-15, -1.1194652e-8, 7.447341e-32, -0.0, -14247490.0, -8.55e-43, -4576.872, 0.009181444], [4.926e-42, -9.267065e-15, -0.0, 7e-45, -0.00036999694, -0.0, -5.7810876e-8, 1.9273644e-21])
diff --git a/rust/lance-linalg/src/distance.rs b/rust/lance-linalg/src/distance.rs
index 23d1cae2d63..142ec1303b4 100644
--- a/rust/lance-linalg/src/distance.rs
+++ b/rust/lance-linalg/src/distance.rs
@@ -25,12 +25,104 @@ pub mod l2;
 pub mod l2_u8;
 pub mod norm_l2;
 
+/// What a per-batch distance kernel yields.
+///
+/// The two batch paths produce different shapes: the build-baseline path maps
+/// lazily over the batch and allocates nothing, while a `#[target_feature]`
+/// kernel must collect eagerly because it cannot be inlined into a lazy
+/// closure. A concrete enum keeps both statically dispatched.
+///
+/// Only sub-AVX2 builds need this. On an AVX2-baseline build the batch methods
+/// return the bare `Map` instead, because any wrapper — trait object or enum —
+/// loses `TrustedLen` (so `.collect()` stops preallocating) and loses
+/// `Map::fold`'s inlined loop. Benchmarks showed that costing 2.5x on the dim-8
+/// batch, far more than the per-vector dispatch it was meant to remove.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+pub(crate) enum BatchIter<L> {
+    /// Lazy per-vector map. No allocation.
+    Lazy(L),
+    /// Eagerly collected by a `#[target_feature]` kernel.
+    Eager(std::vec::IntoIter<f32>),
+}
+
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+impl<L: Iterator<Item = f32>> Iterator for BatchIter<L> {
+    type Item = f32;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            Self::Lazy(iter) => iter.next(),
+            Self::Eager(iter) => iter.next(),
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        match self {
+            Self::Lazy(iter) => iter.size_hint(),
+            Self::Eager(iter) => iter.size_hint(),
+        }
+    }
+
+    /// Delegated, not defaulted. `Map` overrides `fold` to drive the underlying
+    /// `ChunksExact` in one inlined, auto-vectorized loop; the default `fold`
+    /// would instead call `next()` per element, paying an enum branch and
+    /// losing that loop. On the dim-8 batch that costs ~2.5x.
+    #[inline]
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        match self {
+            Self::Lazy(iter) => iter.fold(init, f),
+            Self::Eager(iter) => iter.fold(init, f),
+        }
+    }
+
+    /// `for_each`, `sum` and `collect` all route through `fold`, so delegating
+    /// it covers them too. (`try_fold` cannot be overridden on stable: its
+    /// `Try` bound is unstable.)
+    #[inline]
+    fn for_each<F>(self, f: F)
+    where
+        F: FnMut(Self::Item),
+    {
+        match self {
+            Self::Lazy(iter) => iter.for_each(f),
+            Self::Eager(iter) => iter.for_each(f),
+        }
+    }
+}
+
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+impl<L: ExactSizeIterator<Item = f32>> ExactSizeIterator for BatchIter<L> {
+    #[inline]
+    fn len(&self) -> usize {
+        match self {
+            Self::Lazy(iter) => iter.len(),
+            Self::Eager(iter) => iter.len(),
+        }
+    }
+}
+
 pub use cosine::*;
 pub use dot::*;
 pub use hamming::{
-    Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges, cluster_pairwise_result,
-    extract_hashes_from_fixed_list, hamming_distance_arrow_batch, hamming_u64,
-    pairwise_hamming_distance, pairwise_hamming_distance_parallel,
+    BinaryHashValues, Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges,
+    cluster_pairwise_result, extract_binary_hashes_from_fixed_list, extract_hashes_from_fixed_list,
+    hamming_distance_arrow_batch, hamming_u64, pairwise_hamming_distance,
+    pairwise_hamming_distance_binary, pairwise_hamming_distance_binary_parallel,
+    pairwise_hamming_distance_parallel,
 };
 pub use l2::*;
 use lance_core::deepsize::DeepSizeOf;
diff --git a/rust/lance-linalg/src/distance/cosine.rs b/rust/lance-linalg/src/distance/cosine.rs
index 995191b77eb..457c0c0dd6a 100644
--- a/rust/lance-linalg/src/distance/cosine.rs
+++ b/rust/lance-linalg/src/distance/cosine.rs
@@ -17,12 +17,12 @@ use arrow_array::{
 use arrow_schema::DataType;
 use half::{bf16, f16};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+#[allow(unused_imports)]
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 
 use super::{Dot, norm_l2::norm_l2};
 use super::{Normalize, dot::dot};
+#[allow(unused_imports)]
 use crate::simd::{
     FloatSimd, SIMD,
     f32::{f32x8, f32x16},
@@ -127,6 +127,10 @@ impl Cosine for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::cosine_bf16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels in `bf16_kernel::*` are compiled with
+            // `-march=haswell` minimum (which requires AVX2), so they cannot
+            // run on AVX-only or AVX+FMA hosts. Scalar is the correct route.
             _ => cosine_scalar(x, x_norm, y),
         }
     }
@@ -179,88 +183,348 @@ impl Cosine for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::cosine_f16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => cosine_scalar(x, x_norm, y),
         }
     }
 }
 
-/// f32 kernels for Cosine
+/// f32 single-vector cosine helpers used by `cosine_batch` for fixed
+/// dimensions 8 and 16.
+///
+/// These were previously a single generic `cosine_once<S, N>` but the
+/// monomorphizations have to dispatch on `SIMD_SUPPORT` for the SIMD path
+/// to stay correct under any compile baseline. Splitting them into two
+/// concrete entry points keeps the dispatch site flat and lets each width
+/// route to a `#[target_feature]` AVX2 inner function.
 mod f32 {
     use super::*;
 
-    // TODO: how can we explicitly infer N?
     #[inline]
-    pub(super) fn cosine_once<S: SIMD<f32, N>, const N: usize>(
+    pub(super) fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                    cosine_once_x86::cosine_once_8_avx512(x, x_norm, y)
+                },
+                SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                    cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y)
+                },
+                SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) },
+                _ => cosine_once_8_scalar(x, x_norm, y),
+            }
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            cosine_once_8_other(x, x_norm, y)
+        }
+    }
+
+    #[inline]
+    pub(super) fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                    cosine_once_x86::cosine_once_16_avx512(x, x_norm, y)
+                },
+                SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                    cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y)
+                },
+                SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) },
+                _ => cosine_once_16_scalar(x, x_norm, y),
+            }
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            cosine_once_16_other(x, x_norm, y)
+        }
+    }
+
+    /// Portable scalar `cosine_once` for length-8 vectors. Matches the SIMD
+    /// path modulo summation order.
+    #[cfg(target_arch = "x86_64")]
+    #[inline]
+    pub(super) fn cosine_once_8_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let mut xy = 0.0f32;
+        let mut y2 = 0.0f32;
+        for i in 0..8 {
+            xy += x[i] * y[i];
+            y2 += y[i] * y[i];
+        }
+        1.0 - xy / x_norm / y2.sqrt()
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[inline]
+    pub(super) fn cosine_once_16_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let mut xy = 0.0f32;
+        let mut y2 = 0.0f32;
+        for i in 0..16 {
+            xy += x[i] * y[i];
+            y2 += y[i] * y[i];
+        }
+        1.0 - xy / x_norm / y2.sqrt()
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub(super) mod cosine_once_x86 {
+        use std::arch::x86_64::*;
+
+        use super::{f32x8, f32x16};
+        use crate::simd::SIMD;
+
+        /// AVX + FMA path for 8-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+        #[target_feature(enable = "avx,fma")]
+        pub unsafe fn cosine_once_8_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX + FMA path for 16-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+        #[target_feature(enable = "avx,fma")]
+        pub unsafe fn cosine_once_16_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-only path for 8-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge.
+        #[target_feature(enable = "avx")]
+        pub unsafe fn cosine_once_8_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-only path for 16-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge.
+        #[target_feature(enable = "avx")]
+        pub unsafe fn cosine_once_16_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-512 path for 8-lane cosine: masked load into a `__m512` lower half, reduce.
+        #[target_feature(enable = "avx512f")]
+        pub unsafe fn cosine_once_8_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            // mask 0x00FF: load the lower 8 f32 lanes, zero the upper 8.
+            let mask: __mmask16 = 0x00FF;
+            let xv = _mm512_maskz_loadu_ps(mask, x.as_ptr());
+            let yv = _mm512_maskz_loadu_ps(mask, y.as_ptr());
+            let xy = _mm512_mul_ps(xv, yv);
+            let y2 = _mm512_mul_ps(yv, yv);
+            let xy_sum = _mm512_reduce_add_ps(xy);
+            let y2_sum = _mm512_reduce_add_ps(y2);
+            1.0 - xy_sum / x_norm / y2_sum.sqrt()
+        }
+
+        /// AVX-512 path for 16-lane cosine: single full-width `__m512` load (16 f32 fits one `zmm`).
+        #[target_feature(enable = "avx512f")]
+        pub unsafe fn cosine_once_16_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = _mm512_loadu_ps(x.as_ptr());
+            let yv = _mm512_loadu_ps(y.as_ptr());
+            let xy = _mm512_mul_ps(xv, yv);
+            let y2 = _mm512_mul_ps(yv, yv);
+            let xy_sum = _mm512_reduce_add_ps(xy);
+            let y2_sum = _mm512_reduce_add_ps(y2);
+            1.0 - xy_sum / x_norm / y2_sum.sqrt()
+        }
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[inline]
+    fn cosine_once_8_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let xv = unsafe { f32x8::load_unaligned(x.as_ptr()) };
+        let yv = unsafe { f32x8::load_unaligned(y.as_ptr()) };
+        let y2 = yv * yv;
+        let xy = xv * yv;
+        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[inline]
+    fn cosine_once_16_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let xv = unsafe { f32x16::load_unaligned(x.as_ptr()) };
+        let yv = unsafe { f32x16::load_unaligned(y.as_ptr()) };
+        let y2 = yv * yv;
+        let xy = xv * yv;
+        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+    }
+
+    /// Batch-level SIMD dispatch: the tier is chosen once by the caller, and the
+    /// whole `chunks_exact` loop runs inside one `#[target_feature]` context so
+    /// the per-vector `cosine_once_*` / `cosine_fast` kernels inline (no
+    /// per-vector `*SIMD_SUPPORT` branch, no per-vector call boundary). Used for
+    /// the AVX-512 path on the default wheel and for all tiers on sub-AVX2 builds.
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn cosine_batch_avx_fma(
         x: &[f32],
         x_norm: f32,
-        y: &[f32],
-    ) -> f32 {
-        let x = unsafe { S::load_unaligned(x.as_ptr()) };
-        let y = unsafe { S::load_unaligned(y.as_ptr()) };
-        let y2 = y * y;
-        let xy = x * y;
-        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx_fma(x, x_norm, y) })
+                .collect(),
+        }
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn cosine_batch_avx512(
+        x: &[f32],
+        x_norm: f32,
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx512(x, x_norm, y) })
+                .collect(),
+        }
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn cosine_batch_avx(
+        x: &[f32],
+        x_norm: f32,
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx(x, x_norm, y) })
+                .collect(),
+        }
     }
 }
 
-impl Cosine for f32 {
+/// Inlined f32 cosine kernels for builds whose baseline already guarantees AVX2
+/// (the default `haswell` wheel). No `#[target_feature]`, no runtime dispatch:
+/// under `target-feature=+avx2,+fma` these compile to AVX2 and inline into the
+/// batch loop exactly like the pre-PR code, so the modern path is not taxed by
+/// the runtime-dispatch machinery (only needed below the AVX2 baseline).
+#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
+mod f32_baseline {
+    use super::{dot, f32x8, f32x16, norm_l2};
+    use crate::simd::{FloatSimd, SIMD};
+
     #[inline]
-    fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 {
-        let dim = x.len();
-        let unrolled_len = dim / 16 * 16;
-        let mut y_norm16 = f32x16::zeros();
-        let mut xy16 = f32x16::zeros();
-        for i in (0..unrolled_len).step_by(16) {
-            unsafe {
-                let x = f32x16::load_unaligned(x.as_ptr().add(i));
-                let y = f32x16::load_unaligned(other.as_ptr().add(i));
-                xy16.multiply_add(x, y);
-                y_norm16.multiply_add(y, y);
-            }
+    pub fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
         }
-        let aligned_len = dim / 8 * 8;
-        let mut y_norm8 = f32x8::zeros();
-        let mut xy8 = f32x8::zeros();
-        for i in (unrolled_len..aligned_len).step_by(8) {
-            unsafe {
-                let x = f32x8::load_unaligned(x.as_ptr().add(i));
-                let y = f32x8::load_unaligned(other.as_ptr().add(i));
-                xy8.multiply_add(x, y);
-                y_norm8.multiply_add(y, y);
-            }
+    }
+
+    #[inline]
+    pub fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
         }
-        let y_norm =
-            y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
-        let xy =
-            xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
-        1.0 - xy / x_norm / y_norm.sqrt()
     }
 
     #[inline]
-    fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self {
-        let dim = x.len();
-        let unrolled_len = dim / 16 * 16;
-        let mut xy16 = f32x16::zeros();
-        for i in (0..unrolled_len).step_by(16) {
-            unsafe {
-                let x = f32x16::load_unaligned(x.as_ptr().add(i));
-                let y = f32x16::load_unaligned(y.as_ptr().add(i));
-                xy16.multiply_add(x, y);
+    pub fn cosine_fast(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        unsafe {
+            let dim = x.len();
+            let unrolled_len = dim / 16 * 16;
+            let mut y_norm16 = f32x16::zeros();
+            let mut xy16 = f32x16::zeros();
+            for i in (0..unrolled_len).step_by(16) {
+                let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+                let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+                xy16.multiply_add(xv, yv);
+                y_norm16.multiply_add(yv, yv);
             }
-        }
-        let aligned_len = dim / 8 * 8;
-        let mut xy8 = f32x8::zeros();
-        for i in (unrolled_len..aligned_len).step_by(8) {
-            unsafe {
-                let x = f32x8::load_unaligned(x.as_ptr().add(i));
-                let y = f32x8::load_unaligned(y.as_ptr().add(i));
-                xy8.multiply_add(x, y);
+            let aligned_len = dim / 8 * 8;
+            let mut y_norm8 = f32x8::zeros();
+            let mut xy8 = f32x8::zeros();
+            for i in (unrolled_len..aligned_len).step_by(8) {
+                let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+                let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+                xy8.multiply_add(xv, yv);
+                y_norm8.multiply_add(yv, yv);
             }
+            let y_norm = y_norm16.reduce_sum()
+                + y_norm8.reduce_sum()
+                + norm_l2(&other[aligned_len..]).powi(2);
+            let xy = xy16.reduce_sum()
+                + xy8.reduce_sum()
+                + dot(&x[aligned_len..], &other[aligned_len..]);
+            1.0 - xy / x_norm / y_norm.sqrt()
         }
-        let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
-        1.0 - xy / x_norm / y_norm
+    }
+}
+
+impl Cosine for f32 {
+    #[inline]
+    fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 {
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_fast_f32_dispatched(x, x_norm, other)
     }
 
+    #[inline]
+    fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self {
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_with_norms_f32_dispatched(x, x_norm, y_norm, y)
+    }
+
+    #[allow(unreachable_code)]
     fn cosine_batch<'a>(
         x: &'a [Self],
         batch: &'a [Self],
@@ -268,16 +532,83 @@ impl Cosine for f32 {
     ) -> Box<dyn Iterator<Item = f32> + 'a> {
         let x_norm = norm_l2(x);
 
+        // On a build whose baseline already guarantees AVX2 (the default
+        // `haswell` wheel), avoid the per-vector runtime dispatch + `#[target_feature]`
+        // wrapping that taxes the modern path. Dispatch ONCE per batch: AVX-512
+        // hosts get the wide kernel; everyone else uses the inlined AVX2 baseline
+        // path (base-equivalent). The runtime-dispatch path below is only
+        // compiled/reached when the baseline is below AVX2 (pre-Haswell builds).
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
+        {
+            // dim 8/16 always use the inlined AVX2 baseline: AVX-512 gives no
+            // benefit for such tiny vectors (a masked 512-bit load is slower than
+            // a plain AVX2 load) and only adds dispatch + eager-collect overhead.
+            // Only the larger-dim path routes to AVX-512 on capable hosts — that's
+            // where the wider lanes actually pay off.
+            return match dimension {
+                8 => Box::new(
+                    batch
+                        .chunks_exact(8)
+                        .map(move |y| f32_baseline::cosine_once_8(x, x_norm, y)),
+                ),
+                16 => Box::new(
+                    batch
+                        .chunks_exact(16)
+                        .map(move |y| f32_baseline::cosine_once_16(x, x_norm, y)),
+                ),
+                _ => {
+                    if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) {
+                        Box::new(
+                            unsafe { f32::cosine_batch_avx512(x, x_norm, batch, dimension) }
+                                .into_iter(),
+                        )
+                    } else {
+                        Box::new(
+                            batch
+                                .chunks_exact(dimension)
+                                .map(move |y| f32_baseline::cosine_fast(x, x_norm, y)),
+                        )
+                    }
+                }
+            };
+        }
+
+        // Sub-AVX2 / non-x86 build: hoisted per-batch runtime dispatch.
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx512(x, x_norm, batch, dimension) }
+                            .into_iter(),
+                    );
+                }
+                SimdSupport::Avx2 | SimdSupport::AvxFma => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx_fma(x, x_norm, batch, dimension) }
+                            .into_iter(),
+                    );
+                }
+                SimdSupport::Avx => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx(x, x_norm, batch, dimension) }.into_iter(),
+                    );
+                }
+                _ => {}
+            }
+        }
+
+        // Scalar / non-x86 fallback.
         match dimension {
             8 => Box::new(
                 batch
                     .chunks_exact(dimension)
-                    .map(move |y| f32::cosine_once::<f32x8, 8>(x, x_norm, y)),
+                    .map(move |y| f32::cosine_once_8(x, x_norm, y)),
             ),
             16 => Box::new(
                 batch
                     .chunks_exact(dimension)
-                    .map(move |y| f32::cosine_once::<f32x16, 16>(x, x_norm, y)),
+                    .map(move |y| f32::cosine_once_16(x, x_norm, y)),
             ),
             _ => Box::new(
                 batch
@@ -291,34 +622,102 @@ impl Cosine for f32 {
 impl Cosine for f64 {
     #[inline]
     fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 {
-        use crate::simd::f64::{f64x4, f64x8};
-        use crate::simd::{FloatSimd, SIMD};
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_fast_f64_dispatched(x, x_norm, y)
+    }
+}
 
+/// Fast cosine for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`.
+#[inline]
+fn cosine_fast_f64_dispatched(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f64_x86::cosine_fast_avx512(x, x_norm, y)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f64_x86::cosine_fast_avx_fma(x, x_norm, y)
+            },
+            SimdSupport::Avx => unsafe { f64_x86::cosine_fast_avx(x, x_norm, y) },
+            _ => cosine_scalar(x, x_norm, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_fast_f64_simd_other(x, x_norm, y)
+    }
+}
+
+/// AVX2 + FMA implementation of the f64 cosine_fast kernel.
+///
+/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives
+/// in `crate::simd::f64` (which use raw AVX intrinsics) inline correctly
+/// even when the compile baseline does not have AVX2 enabled. Caller must
+/// ensure the host supports AVX2 + FMA.
+#[cfg(target_arch = "x86_64")]
+mod f64_x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_pd;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX-512 path for f64 fast cosine: 8-wide `__m512d` xy/yy with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_fast_avx512(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc_xy = _mm512_setzero_pd();
+        let mut acc_yy = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let xv = _mm512_loadu_pd(x.as_ptr().add(i));
+            let yv = _mm512_loadu_pd(y.as_ptr().add(i));
+            acc_xy = _mm512_fmadd_pd(xv, yv, acc_xy);
+            acc_yy = _mm512_fmadd_pd(yv, yv, acc_yy);
+        }
+
+        let mut xy = _mm512_reduce_add_pd(acc_xy);
+        let mut yy = _mm512_reduce_add_pd(acc_yy);
+        for i in unrolled_len..dim {
+            xy += x[i] * y[i];
+            yy += y[i] * y[i];
+        }
+
+        let y_norm_sq = yy as f32;
+        let xy_f32 = xy as f32;
+        1.0 - xy_f32 / x_norm / y_norm_sq.sqrt()
+    }
+
+    /// AVX + FMA path for f64 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_fast_avx_fma(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
         let dim = x.len();
         let unrolled_len = dim / 8 * 8;
         let mut y_norm8 = f64x8::zeros();
         let mut xy8 = f64x8::zeros();
         for i in (0..unrolled_len).step_by(8) {
-            unsafe {
-                let xv = f64x8::load_unaligned(x.as_ptr().add(i));
-                let yv = f64x8::load_unaligned(y.as_ptr().add(i));
-                xy8.multiply_add(xv, yv);
-                y_norm8.multiply_add(yv, yv);
-            }
+            let xv = f64x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
         }
         let aligned_len = dim / 4 * 4;
         let mut y_norm4 = f64x4::zeros();
         let mut xy4 = f64x4::zeros();
         for i in (unrolled_len..aligned_len).step_by(4) {
-            unsafe {
-                let xv = f64x4::load_unaligned(x.as_ptr().add(i));
-                let yv = f64x4::load_unaligned(y.as_ptr().add(i));
-                xy4.multiply_add(xv, yv);
-                y_norm4.multiply_add(yv, yv);
-            }
+            let xv = f64x4::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x4::load_unaligned(y.as_ptr().add(i));
+            xy4.multiply_add(xv, yv);
+            y_norm4.multiply_add(yv, yv);
         }
-        let tail_y_norm: Self = y[aligned_len..].iter().map(|&v| v * v).sum();
-        let tail_xy: Self = x[aligned_len..]
+        let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+        let tail_xy: f64 = x[aligned_len..]
             .iter()
             .zip(y[aligned_len..].iter())
             .map(|(&a, &b)| a * b)
@@ -328,6 +727,336 @@ impl Cosine for f64 {
         let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32;
         1.0 - xy / x_norm / y_norm_sq.sqrt()
     }
+
+    /// AVX-only path for f64 fast cosine (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration; tail handled inline.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_fast_avx(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 4 * 4;
+
+        let mut acc_xy = _mm256_setzero_pd();
+        let mut acc_yy = _mm256_setzero_pd();
+        for i in (0..aligned_len).step_by(4) {
+            let xv = _mm256_loadu_pd(x.as_ptr().add(i));
+            let yv = _mm256_loadu_pd(y.as_ptr().add(i));
+            acc_xy = _mm256_add_pd(acc_xy, _mm256_mul_pd(xv, yv));
+            acc_yy = _mm256_add_pd(acc_yy, _mm256_mul_pd(yv, yv));
+        }
+
+        let xy_main = hsum256_pd(acc_xy);
+        let yy_main = hsum256_pd(acc_yy);
+
+        let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+        let tail_xy: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        let y_norm_sq = (yy_main + tail_y_norm) as f32;
+        let xy = (xy_main + tail_xy) as f32;
+        1.0 - xy / x_norm / y_norm_sq.sqrt()
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_fast_f64_simd_other(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::{FloatSimd, SIMD};
+
+    let dim = x.len();
+    let unrolled_len = dim / 8 * 8;
+    let mut y_norm8 = f64x8::zeros();
+    let mut xy8 = f64x8::zeros();
+    for i in (0..unrolled_len).step_by(8) {
+        unsafe {
+            let xv = f64x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+    }
+    let aligned_len = dim / 4 * 4;
+    let mut y_norm4 = f64x4::zeros();
+    let mut xy4 = f64x4::zeros();
+    for i in (unrolled_len..aligned_len).step_by(4) {
+        unsafe {
+            let xv = f64x4::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x4::load_unaligned(y.as_ptr().add(i));
+            xy4.multiply_add(xv, yv);
+            y_norm4.multiply_add(yv, yv);
+        }
+    }
+    let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+    let tail_xy: f64 = x[aligned_len..]
+        .iter()
+        .zip(y[aligned_len..].iter())
+        .map(|(&a, &b)| a * b)
+        .sum();
+
+    let y_norm_sq = (y_norm8.reduce_sum() + y_norm4.reduce_sum() + tail_y_norm) as f32;
+    let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32;
+    1.0 - xy / x_norm / y_norm_sq.sqrt()
+}
+
+/// Cosine for f32 with known norms, runtime-dispatched via `SIMD_SUPPORT`
+/// on x86_64 (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses
+/// the auto-vectorised scalar loop.
+#[inline]
+fn cosine_with_norms_f32_dispatched(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f32_x86::cosine_with_norms_avx512(x, x_norm, y_norm, y)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f32_x86::cosine_with_norms_avx_fma(x, x_norm, y_norm, y)
+            },
+            SimdSupport::Avx => unsafe { f32_x86::cosine_with_norms_avx(x, x_norm, y_norm, y) },
+            _ => cosine_scalar_fast(x, x_norm, y, y_norm),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_with_norms_f32_simd_other(x, x_norm, y_norm, y)
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_with_norms_f32_simd_other(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+    let dim = x.len();
+    let unrolled_len = dim / 16 * 16;
+    let mut xy16 = f32x16::zeros();
+    for i in (0..unrolled_len).step_by(16) {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(y.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+        }
+    }
+    let aligned_len = dim / 8 * 8;
+    let mut xy8 = f32x8::zeros();
+    for i in (unrolled_len..aligned_len).step_by(8) {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+        }
+    }
+    let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
+    1.0 - xy / x_norm / y_norm
+}
+
+/// Fast cosine for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// `simd::f32` primitives, unconditionally backed by NEON / LSX.
+#[inline]
+fn cosine_fast_f32_dispatched(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f32_x86::cosine_fast_avx512(x, x_norm, other)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f32_x86::cosine_fast_avx_fma(x, x_norm, other)
+            },
+            SimdSupport::Avx => unsafe { f32_x86::cosine_fast_avx(x, x_norm, other) },
+            _ => cosine_scalar(x, x_norm, other),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_fast_f32_simd_other(x, x_norm, other)
+    }
+}
+
+/// AVX2 + FMA implementation of the f32 fast cosine kernel.
+///
+/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives
+/// in `crate::simd::f32` (which use raw AVX intrinsics) inline correctly
+/// even when the compile baseline does not have AVX2 enabled. Caller must
+/// ensure the host supports AVX2 + FMA.
+#[cfg(target_arch = "x86_64")]
+mod f32_x86 {
+    use std::arch::x86_64::*;
+
+    use super::{dot, f32x8, f32x16, norm_l2};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX + FMA path for f32 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_fast_avx_fma(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+        let mut y_norm16 = f32x16::zeros();
+        let mut xy16 = f32x16::zeros();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+            y_norm16.multiply_add(yv, yv);
+        }
+        let aligned_len = dim / 8 * 8;
+        let mut y_norm8 = f32x8::zeros();
+        let mut xy8 = f32x8::zeros();
+        for i in (unrolled_len..aligned_len).step_by(8) {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+        let y_norm =
+            y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
+        let xy =
+            xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
+        1.0 - xy / x_norm / y_norm.sqrt()
+    }
+
+    /// AVX-only path for f32 fast cosine (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`/`norm_l2`.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_fast_avx(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 8 * 8;
+
+        let mut acc_xy = _mm256_setzero_ps();
+        let mut acc_yy = _mm256_setzero_ps();
+        for i in (0..aligned_len).step_by(8) {
+            let xv = _mm256_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm256_loadu_ps(other.as_ptr().add(i));
+            acc_xy = _mm256_add_ps(acc_xy, _mm256_mul_ps(xv, yv));
+            acc_yy = _mm256_add_ps(acc_yy, _mm256_mul_ps(yv, yv));
+        }
+
+        let xy_main = hsum256_ps(acc_xy);
+        let yy_main = hsum256_ps(acc_yy);
+
+        let y_norm = yy_main + norm_l2(&other[aligned_len..]).powi(2);
+        let xy = xy_main + dot(&x[aligned_len..], &other[aligned_len..]);
+        1.0 - xy / x_norm / y_norm.sqrt()
+    }
+
+    /// AVX-512 path for f32 fast cosine: 16-wide `__m512` xy/yy with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_fast_avx512(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc_xy = _mm512_setzero_ps();
+        let mut acc_yy = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = _mm512_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm512_loadu_ps(other.as_ptr().add(i));
+            acc_xy = _mm512_fmadd_ps(xv, yv, acc_xy);
+            acc_yy = _mm512_fmadd_ps(yv, yv, acc_yy);
+        }
+
+        let mut xy = _mm512_reduce_add_ps(acc_xy);
+        let mut yy = _mm512_reduce_add_ps(acc_yy);
+        for i in unrolled_len..dim {
+            xy += x[i] * other[i];
+            yy += other[i] * other[i];
+        }
+
+        1.0 - xy / x_norm / yy.sqrt()
+    }
+
+    /// AVX-512 path for f32 cosine with known norms: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_with_norms_avx512(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = _mm512_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm512_loadu_ps(y.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(xv, yv, acc);
+        }
+
+        let mut xy = _mm512_reduce_add_ps(acc);
+        for i in unrolled_len..dim {
+            xy += x[i] * y[i];
+        }
+
+        1.0 - xy / x_norm / y_norm
+    }
+
+    /// AVX + FMA path for f32 cosine with known norms. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_with_norms_avx_fma(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+        let mut xy16 = f32x16::zeros();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(y.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+        }
+        let aligned_len = dim / 8 * 8;
+        let mut xy8 = f32x8::zeros();
+        for i in (unrolled_len..aligned_len).step_by(8) {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+        }
+        let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
+        1.0 - xy / x_norm / y_norm
+    }
+
+    /// AVX-only path for f32 cosine with known norms (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_with_norms_avx(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..aligned_len).step_by(8) {
+            let xv = _mm256_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(xv, yv));
+        }
+
+        let xy_main = hsum256_ps(acc);
+        let xy = xy_main + dot(&x[aligned_len..], &y[aligned_len..]);
+        1.0 - xy / x_norm / y_norm
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_fast_f32_simd_other(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+    let dim = x.len();
+    let unrolled_len = dim / 16 * 16;
+    let mut y_norm16 = f32x16::zeros();
+    let mut xy16 = f32x16::zeros();
+    for i in (0..unrolled_len).step_by(16) {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+            y_norm16.multiply_add(yv, yv);
+        }
+    }
+    let aligned_len = dim / 8 * 8;
+    let mut y_norm8 = f32x8::zeros();
+    let mut xy8 = f32x8::zeros();
+    for i in (unrolled_len..aligned_len).step_by(8) {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+    }
+    let y_norm =
+        y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
+    let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
+    1.0 - xy / x_norm / y_norm.sqrt()
 }
 
 /// Fallback non-SIMD implementation
@@ -437,6 +1166,25 @@ pub fn cosine_distance_arrow_batch(
     }
 }
 
+/// Portable scalar reference cosine over f64 inputs. Used by parity tests
+/// to compare against every dispatched per-tier inner kernel. Computes
+/// `1 - xy / (x_norm * y_norm_sq.sqrt())` in f64 then casts to f32, matching
+/// the reduction order of the dispatched kernels.
+#[cfg(test)]
+fn cosine_fast_scalar(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum();
+    let y_norm_sq: f64 = y.iter().map(|&v| v * v).sum();
+    1.0 - (xy as f32) / x_norm / (y_norm_sq as f32).sqrt()
+}
+
+/// Portable scalar reference cosine when both norms are known. Mirrors
+/// `cosine_with_norms_f32_dispatched` for parity testing.
+#[cfg(test)]
+fn cosine_with_norms_scalar(x: &[f64], x_norm: f32, y_norm: f32, y: &[f64]) -> f32 {
+    let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum();
+    1.0 - (xy as f32) / x_norm / y_norm
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -559,5 +1307,447 @@ mod tests {
             prop_assume!(norm_l2(&y) > 1e-20);
             do_cosine_test(&x, &y)?;
         }
+
+        /// Cross-backend parity for the f32 cosine_fast kernel. Exercises the
+        /// scalar fallback (`cosine_scalar`) against the dispatched SIMD path
+        /// so the runtime fallback is exercised even on AVX2-capable CI hosts.
+        #[test]
+        fn test_cosine_fast_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = <f32 as Cosine>::cosine_fast(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f32 cosine_fast kernel. Early-returns
+        /// on hosts without AVX-512F so the test stays portable.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx512 = unsafe { f32_x86::cosine_fast_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f32 cosine_fast kernel. Covers
+        /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns
+        /// on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx_fma = unsafe { f32_x86::cosine_fast_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f32 cosine_fast kernel. Covers
+        /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on
+        /// hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx = unsafe { f32_x86::cosine_fast_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Cross-backend parity for the f32 cosine_with_norms kernel.
+        /// Exercises the scalar fallback (`cosine_scalar_fast`) against the
+        /// dispatched SIMD path so the runtime fallback is exercised even on
+        /// AVX2-capable CI hosts.
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_with_norms_scalar(&x_f64, x_norm, y_norm, &y_f64);
+            let simd = <f32 as Cosine>::cosine_with_norms(&x, x_norm, y_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f32 cosine_with_norms kernel.
+        /// Early-returns on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx512 = unsafe { f32_x86::cosine_with_norms_avx512(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f32 cosine_with_norms kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx_fma = unsafe { f32_x86::cosine_with_norms_avx_fma(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f32 cosine_with_norms kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx = unsafe { f32_x86::cosine_with_norms_avx(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Cross-backend parity for the f64 cosine_fast kernel. Uses the
+        /// hand-rolled `cosine_fast_scalar` (not the trait-routed
+        /// `cosine_scalar`, which would itself dispatch through `dot::<f64>`)
+        /// so the reference stays free of any AVX path on AVX2-capable hosts.
+        #[test]
+        fn test_cosine_fast_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let simd = <f64 as Cosine>::cosine_fast(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f64 cosine_fast kernel. Early-returns
+        /// on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx512 = unsafe { f64_x86::cosine_fast_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f64 cosine_fast kernel. Covers
+        /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns
+        /// on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx_fma = unsafe { f64_x86::cosine_fast_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f64 cosine_fast kernel. Covers
+        /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on
+        /// hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx = unsafe { f64_x86::cosine_fast_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Parity check for `cosine_once_8` (despecialised 8-lane width).
+        ///
+        /// The `epsilon = 1e-6` clause handles the case where the proptest
+        /// generator produces inputs with extreme dynamic range (e.g., mixing
+        /// `1e-43` with `1e7` in the same vector). When the dot product is
+        /// dominated by one large term and the cosine result is near zero,
+        /// the f32-precision SIMD path and the f64-precision scalar reference
+        /// can legitimately differ by more than `max_relative = 1e-3` of the
+        /// (near-zero) result. The absolute epsilon catches these without
+        /// masking real bugs (where the absolute error would be > 1e-6).
+        #[test]
+        fn test_cosine_once_8_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = f32::cosine_once_8(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6));
+        }
+
+        /// Parity check for `cosine_once_16` (despecialised 16-lane width).
+        /// See `test_cosine_once_8_scalar_simd_parity` for `epsilon` rationale.
+        #[test]
+        fn test_cosine_once_16_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = f32::cosine_once_16(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6));
+        }
+
+        /// AVX-512-direct parity for the 8-lane cosine_once kernel. Verifies
+        /// the masked-load (mask 0x00FF) AVX-512 implementation produces
+        /// the same result as the scalar reference. Early-returns on hosts
+        /// without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx512 =
+                unsafe { super::f32::cosine_once_x86::cosine_once_8_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX-512-direct parity for the 16-lane cosine_once kernel. Verifies
+        /// the full-width `__m512` load implementation produces the same
+        /// result as the scalar reference. Early-returns on hosts without
+        /// AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx512 =
+                unsafe { super::f32::cosine_once_x86::cosine_once_16_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the 8-lane cosine_once kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx_fma =
+                unsafe { super::f32::cosine_once_x86::cosine_once_8_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the 16-lane cosine_once kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx_fma =
+                unsafe { super::f32::cosine_once_x86::cosine_once_16_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the 8-lane cosine_once kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx = unsafe { super::f32::cosine_once_x86::cosine_once_8_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the 16-lane cosine_once kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx = unsafe { super::f32::cosine_once_x86::cosine_once_16_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+    }
+
+    /// Asserts a batch-level f32 cosine SIMD kernel matches the scalar
+    /// `cosine_fast` reference for every vector in a multi-vector batch. Runs
+    /// each of the kernel's three internal dimension arms (8, 16, and the
+    /// general `chunks_exact` path). The batch kernels only run at runtime on
+    /// sub-AVX2 builds, so a direct call is the only way they get covered.
+    #[cfg(target_arch = "x86_64")]
+    fn check_cosine_batch_kernel(kernel: unsafe fn(&[f32], f32, &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let x_norm = norm_l2(&x);
+            let num_vectors = 3;
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, x_norm, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let y_f64: Vec<f64> = chunk.iter().map(|&v| v as f64).collect();
+                let expected = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+                assert_relative_eq!(g, expected, max_relative = 1e-3, epsilon = 1e-6);
+            }
+        }
+    }
+
+    /// AVX + FMA batch kernel parity (AVX2 / AVX+FMA tiers). Runs on any
+    /// Haswell-or-newer host; early-returns without AVX and FMA.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx_fma_matches_scalar() {
+        if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx_fma);
+    }
+
+    /// AVX-only batch kernel parity (Sandy Bridge / Ivy Bridge tier).
+    /// Early-returns on hosts without AVX.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx);
+    }
+
+    /// AVX-512 batch kernel parity. Early-returns on hosts without AVX-512F.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx512);
     }
 }
diff --git a/rust/lance-linalg/src/distance/cosine_u8.rs b/rust/lance-linalg/src/distance/cosine_u8.rs
index b2d06d35c31..59833e020a7 100644
--- a/rust/lance-linalg/src/distance/cosine_u8.rs
+++ b/rust/lance-linalg/src/distance/cosine_u8.rs
@@ -199,6 +199,10 @@ fn select_backend() -> CosineU8AccumFn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::cosine_u8_accum_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor
+        // AVX+FMA provides.
     }
 
     cosine_u8_accum_scalar
diff --git a/rust/lance-linalg/src/distance/dot.rs b/rust/lance-linalg/src/distance/dot.rs
index 5903d24e0e5..3411d7e8dce 100644
--- a/rust/lance-linalg/src/distance/dot.rs
+++ b/rust/lance-linalg/src/distance/dot.rs
@@ -14,12 +14,16 @@ use arrow_schema::DataType;
 use half::{bf16, f16};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
 use lance_core::assume_eq;
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+#[allow(unused_imports)]
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 use num_traits::{AsPrimitive, Num, real::Real};
 
 use crate::Result;
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+use crate::distance::BatchIter;
 
 /// Default implementation of dot product.
 ///
@@ -111,6 +115,24 @@ pub fn dot_distance<T: Dot>(from: &[T], to: &[T]) -> f32 {
 pub trait Dot: Num {
     /// Dot product.
     fn dot(x: &[Self], y: &[Self]) -> f32;
+
+    /// Dot product of `x` against each `dimension`-sized vector in `batch`.
+    ///
+    /// The default calls [`Dot::dot`] per vector. `f32` overrides it so the
+    /// SIMD tier is chosen once for the whole batch instead of once per
+    /// vector — on a build whose baseline already implies AVX2, per-vector
+    /// dispatch costs more than the kernel it selects.
+    ///
+    /// Returns `impl Iterator` rather than a trait object: hot consumers drive
+    /// this one element at a time, so a `Box<dyn Iterator>` would cost a
+    /// virtual call per element and an allocation per batch.
+    fn dot_batch<'a>(
+        x: &'a [Self],
+        batch: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = f32> + 'a {
+        batch.chunks_exact(dimension).map(move |y| Self::dot(x, y))
+    }
 }
 
 #[cfg(feature = "fp16kernels")]
@@ -161,6 +183,9 @@ impl Dot for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::dot_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => dot_scalar::<Self, f32, 32>(x, y),
         }
     }
@@ -214,6 +239,9 @@ impl Dot for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::dot_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => dot_scalar::<Self, f32, 32>(x, y),
         }
     }
@@ -222,10 +250,128 @@ impl Dot for f16 {
 impl Dot for f32 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
-        dot_scalar::<Self, Self, 16>(x, y)
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable
+        // scalar fallback. Same shape as the f64 sibling and the existing
+        // u8 distance kernels in `dot_u8.rs`.
+        dot_f32_dispatched(x, y)
+    }
+
+    fn dot_batch<'a>(
+        x: &'a [Self],
+        batch: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = Self> + 'a {
+        // Exactly one arm compiles. Keeping each a tail expression (rather than
+        // an early `return` guarded by `cfg`) mirrors `dot_f32_dispatched` and
+        // avoids an unreachable tail on AVX2-baseline builds.
+        // AVX2-baseline build (the default `haswell` wheel). Hoist the tier
+        // choice out of the loop, but keep the SIMD kernel: the baseline already
+        // guarantees avx2+fma, so call the AVX+FMA kernel directly rather than
+        // re-checking per vector. Falling back to the scalar kernel here would
+        // lose ~4x at small dimensions, which is where batch calls live (PQ
+        // sub-vectors are 8 wide).
+        //
+        // The iterator is a bare `Map`: `Map<ChunksExact, _>` is `TrustedLen`,
+        // so `.collect()` preallocates, and `Map::fold` drives `ChunksExact` in
+        // one inlined loop. Any wrapper — trait object or enum — loses both.
+        #[cfg(all(
+            target_arch = "x86_64",
+            target_feature = "avx2",
+            target_feature = "fma"
+        ))]
+        {
+            // See `L2::l2_batch` for f32: below 16 lanes `dot_scalar`'s chunking
+            // degenerates to a scalar remainder loop, so the explicit AVX kernel
+            // wins big; above it the autovectorizer is already good and the
+            // 8-wide kernel can lose, so keep the pre-dispatch kernel exactly.
+            //
+            // SAFETY: avx2+fma are enabled for the whole crate by the build
+            // baseline, so the kernel's `#[target_feature]` contract holds
+            // statically.
+            let narrow = dimension <= 16;
+            batch.chunks_exact(dimension).map(move |y| {
+                if narrow {
+                    unsafe { x86::dot_f32_avx_fma(x, y) }
+                } else {
+                    dot_f32_scalar(x, y)
+                }
+            })
+        }
+        #[cfg(all(
+            target_arch = "x86_64",
+            not(all(target_feature = "avx2", target_feature = "fma"))
+        ))]
+        {
+            dot_batch_f32_runtime_dispatch(x, batch, dimension)
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            batch.chunks_exact(dimension).map(move |y| Self::dot(x, y))
+        }
+    }
+}
+
+/// Sub-AVX2 builds: the scalar kernel cannot reach the wide registers, so pick
+/// a `#[target_feature]` kernel — once for the batch, not once per vector.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+#[inline]
+fn dot_batch_f32_runtime_dispatch<'a>(
+    x: &'a [f32],
+    batch: &'a [f32],
+    dimension: usize,
+) -> impl Iterator<Item = f32> + 'a {
+    // SAFETY: each kernel is entered only under its matching runtime detection.
+    match *SIMD_SUPPORT {
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx512(x, batch, dimension) }.into_iter())
+        }
+        SimdSupport::Avx2 | SimdSupport::AvxFma => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx_fma(x, batch, dimension) }.into_iter())
+        }
+        SimdSupport::Avx => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx(x, batch, dimension) }.into_iter())
+        }
+        _ => BatchIter::Lazy(
+            batch
+                .chunks_exact(dimension)
+                .map(move |y| dot_f32_scalar(x, y)),
+        ),
+    }
+}
+
+/// Dot product for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn dot_f32_dispatched(x: &[f32], y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f32_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f32_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::dot_f32_avx(x, y) },
+            _ => dot_f32_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        dot_f32_scalar(x, y)
     }
 }
 
+/// Portable scalar dot product for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn dot_f32_scalar(x: &[f32], y: &[f32]) -> f32 {
+    dot_scalar::<f32, f32, 16>(x, y)
+}
+
 impl Dot for f64 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
@@ -233,9 +379,246 @@ impl Dot for f64 {
     }
 }
 
-/// Explicit SIMD dot product for f64.
+/// Dot product for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX.
 #[inline]
 fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f64_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f64_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::dot_f64_avx(x, y) },
+            _ => dot_f64_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        dot_f64_simd_other(x, y)
+    }
+}
+
+/// Portable scalar dot product for f64. Used as the x86_64 fallback when no
+/// AVX2 is detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn dot_f64_scalar(x: &[f64], y: &[f64]) -> f32 {
+    x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::<f64>() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// Dot product of `x` against every `dimension`-sized vector in `batch`,
+    /// entering the AVX-512 tier once for the whole batch rather than once per
+    /// vector.
+    ///
+    /// # Safety
+    /// The host must support AVX-512F.
+    ///
+    /// Only compiled for builds whose baseline is below avx2+fma; at or above
+    /// that baseline `dot_batch` inlines the kernel directly and never runtime-
+    /// dispatches, so this wrapper would be dead code (see `dot_batch`).
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn dot_batch_f32_avx512(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx512(x, y) })
+            .collect()
+    }
+
+    /// As [`dot_batch_f32_avx512`], for the AVX2 and AVX+FMA tiers.
+    ///
+    /// # Safety
+    /// The host must support AVX and FMA.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn dot_batch_f32_avx_fma(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx_fma(x, y) })
+            .collect()
+    }
+
+    /// As [`dot_batch_f32_avx512`], for the AVX-without-FMA tier.
+    ///
+    /// # Safety
+    /// The host must support AVX.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn dot_batch_f32_avx(x: &[f32], batch: &[f32], dimension: usize) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx(x, y) })
+            .collect()
+    }
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn dot_f64_avx512(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm512_loadu_pd(x.as_ptr().add(i));
+            let b = _mm512_loadu_pd(y.as_ptr().add(i));
+            acc = _mm512_fmadd_pd(a, b, acc);
+        }
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (_mm512_reduce_add_pd(acc) + tail) as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn dot_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            acc8.multiply_add(a, b);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            acc4.multiply_add(a, b);
+        }
+
+        let tail: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn dot_f64_avx(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let a = _mm256_loadu_pd(x.as_ptr().add(i));
+            let b = _mm256_loadu_pd(y.as_ptr().add(i));
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(a, b));
+        }
+
+        // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (acc_sum + tail) as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn dot_f32_avx512(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let a = _mm512_loadu_ps(x.as_ptr().add(i));
+            let b = _mm512_loadu_ps(y.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(a, b, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        _mm512_reduce_add_ps(acc) + tail
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn dot_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_fmadd_ps(a, b, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+
+    /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn dot_f32_avx(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(a, b));
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn dot_f64_simd_other(x: &[f64], y: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -285,7 +668,7 @@ pub fn dot_distance_batch<'a, T: Dot>(
 ) -> Box<dyn Iterator<Item = f32> + 'a> {
     assume_eq!(from.len(), dimension);
     assume_eq!(to.len() % dimension, 0);
-    Box::new(to.chunks_exact(dimension).map(|v| dot_distance(from, v)))
+    Box::new(T::dot_batch(from, to, dimension).map(|d| 1.0 - d))
 }
 
 fn do_dot_distance_arrow_batch<T: ArrowFloatType>(
@@ -309,10 +692,9 @@ where
                 to.value_type()
             )))?;
 
-    let dists = to_values
-        .as_slice()
-        .chunks_exact(dimension)
-        .map(|v| dot_distance(from.as_slice(), v));
+    // Route through `dot_distance_batch` rather than mapping `dot_distance` per
+    // vector, so this entry point gets the same hoisted dispatch.
+    let dists = dot_distance_batch(from.as_slice(), to_values.as_slice(), dimension);
 
     Ok(Arc::new(Float32Array::new(
         dists.collect(),
@@ -480,5 +862,264 @@ mod tests {
         fn test_dot_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){
             do_dot_test(&x, &y)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `dot_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            let scalar = dot_f64_scalar(&x, &y);
+            let simd = dot_f64_simd(&x, &y);
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error));
+        }
+
+        /// Parity check for `dot_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar dot path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep
+        /// this test architecture-agnostic (the x86_64-only `dot_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_dot_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = x_f64
+                .iter()
+                .zip(y_f64.iter())
+                .map(|(&a, &b)| a * b)
+                .sum::<f64>() as f32;
+            let simd = <f32 as Dot>::dot(&x, &y);
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error));
+        }
+
+        /// AVX-512-direct parity for f32: explicitly compares the scalar
+        /// fallback against the native f32 AVX-512 inner kernel on
+        /// AVX-512F-capable hosts. Early-returns on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx512 = unsafe { x86::dot_f32_avx512(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error));
+        }
+
+        /// AVX + FMA-direct parity for the f32 dot kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx_fma = unsafe { x86::dot_f32_avx_fma(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error));
+        }
+
+        /// AVX-only-direct parity for the f32 dot kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx = unsafe { x86::dot_f32_avx(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx512 = unsafe { x86::dot_f64_avx512(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error));
+        }
+
+        /// AVX + FMA-direct parity for the f64 dot kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx_fma = unsafe { x86::dot_f64_avx_fma(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error));
+        }
+
+        /// AVX-only-direct parity for the f64 dot kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier (AVX without FMA). Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx = unsafe { x86::dot_f64_avx(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error));
+        }
+    }
+
+    /// `dot_batch` must agree with the per-vector `dot` it replaced, on every
+    /// build: AVX2-baseline, hoisted-dispatch, and portable fallback all
+    /// funnel through here.
+    #[rstest::rstest]
+    #[case::dim_8(8)]
+    #[case::dim_16(16)]
+    #[case::dim_32(32)]
+    #[case::dim_1024(1024)]
+    fn test_dot_batch_f32_matches_per_vector_dot(#[case] dimension: usize) {
+        let num_vectors = 5;
+        let x: Vec<f32> = (0..dimension)
+            .map(|i| ((i % 13) as f32) * 0.25 + 1.0)
+            .collect();
+        let batch: Vec<f32> = (0..dimension * num_vectors)
+            .map(|i| ((i % 11) as f32) * 0.5 - 2.0)
+            .collect();
+
+        let got: Vec<f32> = f32::dot_batch(&x, &batch, dimension).collect();
+        let want: Vec<f32> = batch
+            .chunks_exact(dimension)
+            .map(|y| f32::dot(&x, y))
+            .collect();
+
+        assert_eq!(got.len(), num_vectors);
+        for (g, w) in got.iter().zip(want.iter()) {
+            assert!(
+                approx::relative_eq!(g, w, epsilon = 1e-4),
+                "dim {dimension}: batch {g} != per-vector {w}"
+            );
+        }
+    }
+
+    /// `dot_distance_batch` still yields `1.0 - dot`, unchanged by the hoist.
+    #[test]
+    fn test_dot_distance_batch_preserves_distance_semantics() {
+        let dimension = 32;
+        let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.1).collect();
+        let batch: Vec<f32> = (0..dimension * 3).map(|i| (i as f32) * 0.05).collect();
+
+        let got: Vec<f32> = dot_distance_batch(&x, &batch, dimension).collect();
+        for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+            assert!(approx::relative_eq!(
+                g,
+                1.0 - f32::dot(&x, chunk),
+                epsilon = 1e-5
+            ));
+        }
+    }
+
+    /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2
+    /// builds or AVX-512 hosts, so call them directly to cover them.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    fn check_dot_batch_kernel(kernel: unsafe fn(&[f32], &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let num_vectors = 3;
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let want = dot_scalar::<f32, f32, 16>(&x, chunk);
+                assert!(
+                    approx::relative_eq!(g, want, epsilon = 1e-4),
+                    "dim {dimension}: kernel {g} != scalar {want}"
+                );
+            }
+        }
+    }
+
+    // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds
+    // (see `x86::dot_batch_f32_avx512`), so gate their tests the same way.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx_fma_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx_fma);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx512);
     }
 }
diff --git a/rust/lance-linalg/src/distance/dot_u8.rs b/rust/lance-linalg/src/distance/dot_u8.rs
index de5522cddfe..7b2e335f094 100644
--- a/rust/lance-linalg/src/distance/dot_u8.rs
+++ b/rust/lance-linalg/src/distance/dot_u8.rs
@@ -134,6 +134,10 @@ fn select_backend() -> DotU8Fn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::dot_u8_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor
+        // AVX+FMA provides.
     }
 
     dot_u8_scalar
diff --git a/rust/lance-linalg/src/distance/hamming.rs b/rust/lance-linalg/src/distance/hamming.rs
index a6f4b038195..1eeb8e42c38 100644
--- a/rust/lance-linalg/src/distance/hamming.rs
+++ b/rust/lance-linalg/src/distance/hamming.rs
@@ -4,7 +4,7 @@
 //! Hamming distance.
 //!
 //! This module provides hamming distance computation for binary vectors,
-//! including SIMD-accelerated pairwise hamming distance for 64-bit hashes.
+//! including SIMD-accelerated pairwise hamming distance for binary hashes.
 
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -102,6 +102,196 @@ pub fn hamming_u64(a: u64, b: u64) -> u32 {
     (a ^ b).count_ones()
 }
 
+/// Binary hash values stored as 64-bit lanes in lane-major order.
+///
+/// For a hash width of `N` bytes, `N` must be a positive multiple of 8 and the
+/// number of lanes is `N / 8`. Lane-major layout keeps each lane contiguous for
+/// all rows, which allows the pairwise path to reuse the SIMD `u64` batch
+/// implementation for wider hashes.
+#[derive(Debug, Clone)]
+pub struct BinaryHashValues {
+    lane_values: Vec<u64>,
+    num_rows: usize,
+    byte_width: usize,
+}
+
+impl BinaryHashValues {
+    /// Create hash values from lane-major `u64` values.
+    pub fn try_new(lane_values: Vec<u64>, num_rows: usize, byte_width: usize) -> Result<Self> {
+        let num_lanes = validate_hash_byte_width(byte_width)?;
+        let expected_values = checked_lane_value_count(num_rows, num_lanes)?;
+        if lane_values.len() != expected_values {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected {} lane values for {} rows and {} byte hashes, got {}",
+                expected_values,
+                num_rows,
+                byte_width,
+                lane_values.len()
+            )));
+        }
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    /// Extract binary hash values from a `FixedSizeList<UInt8, N>` Arrow array.
+    pub fn from_fixed_size_list(array: &FixedSizeListArray) -> Result<Self> {
+        let byte_width = usize::try_from(array.value_length()).map_err(|_| {
+            Error::InvalidArgumentError(format!(
+                "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}",
+                array.value_length()
+            ))
+        })?;
+        let num_lanes = validate_hash_byte_width(byte_width)?;
+
+        let values = array
+            .values()
+            .as_any()
+            .downcast_ref::<arrow_array::UInt8Array>()
+            .ok_or_else(|| {
+                Error::InvalidArgumentError(
+                    "Expected UInt8Array values in FixedSizeList".to_string(),
+                )
+            })?;
+
+        let num_rows = array.len();
+        let value_count = checked_lane_value_count(num_rows, num_lanes)?;
+        let expected_bytes = checked_hash_byte_count(num_rows, byte_width)?;
+        let bytes = values.values();
+        if bytes.len() != expected_bytes {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected {} bytes for {} rows and {} byte hashes, got {}",
+                expected_bytes,
+                num_rows,
+                byte_width,
+                bytes.len()
+            )));
+        }
+
+        let mut lane_values = vec![0u64; value_count];
+        for row in 0..num_rows {
+            let row_start = row * byte_width;
+            for lane in 0..num_lanes {
+                let start = row_start + lane * 8;
+                let mut arr = [0u8; 8];
+                arr.copy_from_slice(&bytes[start..start + 8]);
+                lane_values[lane * num_rows + row] = u64::from_le_bytes(arr);
+            }
+        }
+
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    /// Concatenate chunks with the same hash width into a single lane-major set.
+    pub fn concat(chunks: &[Self]) -> Result<Self> {
+        let Some(first) = chunks.first() else {
+            return Err(Error::InvalidArgumentError(
+                "Cannot concatenate zero binary hash chunks".to_string(),
+            ));
+        };
+
+        let byte_width = first.byte_width;
+        let num_lanes = first.num_lanes();
+        let mut num_rows = 0usize;
+        for chunk in chunks {
+            if chunk.byte_width != byte_width {
+                return Err(Error::InvalidArgumentError(format!(
+                    "Cannot concatenate binary hash chunks with different widths: {} and {} bytes",
+                    byte_width, chunk.byte_width
+                )));
+            }
+            num_rows = num_rows.checked_add(chunk.num_rows).ok_or_else(|| {
+                Error::InvalidArgumentError(
+                    "Binary hash row count overflowed while concatenating chunks".to_string(),
+                )
+            })?;
+        }
+
+        let value_count = checked_lane_value_count(num_rows, num_lanes)?;
+        let mut lane_values = vec![0u64; value_count];
+        let mut row_offset = 0;
+        for chunk in chunks {
+            for lane in 0..num_lanes {
+                let dest_start = lane * num_rows + row_offset;
+                let dest_end = dest_start + chunk.num_rows;
+                lane_values[dest_start..dest_end].copy_from_slice(chunk.lane(lane));
+            }
+            row_offset += chunk.num_rows;
+        }
+
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.num_rows
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.num_rows == 0
+    }
+
+    pub fn byte_width(&self) -> usize {
+        self.byte_width
+    }
+
+    pub fn num_lanes(&self) -> usize {
+        self.byte_width / 8
+    }
+
+    pub fn lane(&self, lane: usize) -> &[u64] {
+        let start = lane * self.num_rows;
+        &self.lane_values[start..start + self.num_rows]
+    }
+
+    pub fn into_u64_values(self) -> Result<Vec<u64>> {
+        if self.num_lanes() != 1 {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected 8-byte binary hashes, got {} byte hashes",
+                self.byte_width
+            )));
+        }
+        Ok(self.lane_values)
+    }
+}
+
+fn checked_lane_value_count(num_rows: usize, num_lanes: usize) -> Result<usize> {
+    num_rows.checked_mul(num_lanes).ok_or_else(|| {
+        Error::InvalidArgumentError(format!(
+            "Binary hash lane value count overflowed for {} rows and {} lanes",
+            num_rows, num_lanes
+        ))
+    })
+}
+
+fn checked_hash_byte_count(num_rows: usize, byte_width: usize) -> Result<usize> {
+    num_rows.checked_mul(byte_width).ok_or_else(|| {
+        Error::InvalidArgumentError(format!(
+            "Binary hash byte count overflowed for {} rows and {} byte hashes",
+            num_rows, byte_width
+        ))
+    })
+}
+
+fn validate_hash_byte_width(byte_width: usize) -> Result<usize> {
+    if byte_width == 0 || !byte_width.is_multiple_of(8) {
+        return Err(Error::InvalidArgumentError(format!(
+            "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}",
+            byte_width
+        )));
+    }
+    Ok(byte_width / 8)
+}
+
 /// Result of pairwise hamming distance computation.
 #[derive(Debug, Clone)]
 pub struct PairwiseResult {
@@ -386,6 +576,88 @@ pub fn pairwise_hamming_distance_parallel(
     combined
 }
 
+/// Compute pairwise hamming distances for all pairs of fixed-width binary hashes.
+///
+/// This supports any hash width that is a positive multiple of 8 bytes. For
+/// 8-byte hashes this delegates to the existing `u64` implementation.
+pub fn pairwise_hamming_distance_binary(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    if hashes.num_lanes() == 1 {
+        return pairwise_hamming_distance(hashes.lane(0), row_ids, threshold);
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let num_pairs = n * (n - 1) / 2;
+    let mut result = PairwiseResult::with_capacity(num_pairs.min(1_000_000));
+
+    for i in 0..n {
+        for j in (i + 1)..n {
+            let mut dist = 0;
+            for lane in 0..hashes.num_lanes() {
+                let lane_values = hashes.lane(lane);
+                dist += hamming_u64(lane_values[i], lane_values[j]);
+            }
+            if dist <= threshold {
+                let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
+/// Compute pairwise hamming distances in parallel for fixed-width binary hashes.
+///
+/// Wider hashes reuse the SIMD `u64` batch implementation lane-by-lane.
+pub fn pairwise_hamming_distance_binary_parallel(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    if hashes.num_lanes() == 1 {
+        return pairwise_hamming_distance_parallel(hashes.lane(0), row_ids, threshold);
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let total_pairs = n * (n - 1) / 2;
+
+    if total_pairs < 10_000 {
+        return pairwise_hamming_distance_binary(hashes, row_ids, Some(threshold));
+    }
+
+    let threads = rayon::current_num_threads();
+    let pairs_per_chunk = total_pairs.div_ceil(threads);
+    let chunks = compute_balanced_chunks(n, pairs_per_chunk);
+
+    let results: Vec<PairwiseResult> = chunks
+        .into_par_iter()
+        .map(|(start_row, end_row)| {
+            process_row_range_binary(hashes, row_ids, threshold, start_row, end_row)
+        })
+        .collect();
+
+    let mut combined = PairwiseResult::new();
+    for r in results {
+        combined.extend(r);
+    }
+    combined
+}
+
 /// Compute balanced chunks for parallel processing.
 fn compute_balanced_chunks(n: usize, target_pairs_per_chunk: usize) -> Vec<(usize, usize)> {
     let mut chunks = Vec::new();
@@ -439,36 +711,80 @@ fn process_row_range(
     result
 }
 
+/// Process a range of rows for pairwise comparison of wider binary hashes.
+fn process_row_range_binary(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: u32,
+    start_row: usize,
+    end_row: usize,
+) -> PairwiseResult {
+    let n = hashes.len();
+    let mut result = PairwiseResult::new();
+    let mut distances = Vec::new();
+    let mut lane_distances = Vec::new();
+
+    for i in start_row..end_row {
+        let remaining = n - i - 1;
+        if remaining == 0 {
+            continue;
+        }
+
+        distances.clear();
+        distances.resize(remaining, 0);
+
+        let first_lane = hashes.lane(0);
+        hamming_batch_u64(
+            first_lane[i],
+            &first_lane[i + 1..],
+            distances.as_mut_slice(),
+        );
+
+        for lane in 1..hashes.num_lanes() {
+            lane_distances.clear();
+            lane_distances.resize(remaining, 0);
+            let lane_values = hashes.lane(lane);
+            hamming_batch_u64(
+                lane_values[i],
+                &lane_values[i + 1..],
+                lane_distances.as_mut_slice(),
+            );
+            for (distance, lane_distance) in distances.iter_mut().zip(&lane_distances) {
+                *distance += *lane_distance;
+            }
+        }
+
+        let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+        for (j_offset, &dist) in distances.iter().enumerate() {
+            if dist <= threshold {
+                let j = i + 1 + j_offset;
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
 /// Extract u64 hashes from a FixedSizeList<UInt8, 8> Arrow array.
 pub fn extract_hashes_from_fixed_list(array: &FixedSizeListArray) -> Result<Vec<u64>> {
-    let list_size = array.value_length();
-    if list_size != 8 {
+    let hashes = extract_binary_hashes_from_fixed_list(array)?;
+    if hashes.byte_width() != 8 {
         return Err(Error::InvalidArgumentError(format!(
             "Expected FixedSizeList with size 8, got size {}",
-            list_size
+            hashes.byte_width()
         )));
     }
+    hashes.into_u64_values()
+}
 
-    let values = array
-        .values()
-        .as_any()
-        .downcast_ref::<arrow_array::UInt8Array>()
-        .ok_or_else(|| {
-            Error::InvalidArgumentError("Expected UInt8Array values in FixedSizeList".to_string())
-        })?;
-
-    let n = array.len();
-    let mut hashes = Vec::with_capacity(n);
-
-    for i in 0..n {
-        let start = i * 8;
-        let bytes = &values.values()[start..start + 8];
-        let mut arr = [0u8; 8];
-        arr.copy_from_slice(bytes);
-        hashes.push(u64::from_le_bytes(arr));
-    }
-
-    Ok(hashes)
+/// Extract binary hashes from a `FixedSizeList<UInt8, N>` Arrow array where
+/// `N` is a positive multiple of 8 bytes.
+pub fn extract_binary_hashes_from_fixed_list(
+    array: &FixedSizeListArray,
+) -> Result<BinaryHashValues> {
+    BinaryHashValues::from_fixed_size_list(array)
 }
 
 /// Union-Find data structure with path compression for clustering.
@@ -733,6 +1049,7 @@ pub fn cluster_pairwise_result(result: &PairwiseResult) -> ClusteringResult {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use lance_arrow::FixedSizeListArrayExt;
 
     #[test]
     fn test_hamming() {
@@ -912,6 +1229,121 @@ mod tests {
         v
     }
 
+    #[test]
+    fn test_extract_binary_hashes_from_fixed_list_128() {
+        use arrow_array::UInt8Array;
+
+        let rows = [
+            (0x0102_0304_0506_0708u64, 0x1112_1314_1516_1718u64),
+            (0x2122_2324_2526_2728u64, 0x3132_3334_3536_3738u64),
+        ];
+        let bytes: Vec<u8> = rows
+            .iter()
+            .flat_map(|(lo, hi)| lo.to_le_bytes().into_iter().chain(hi.to_le_bytes()))
+            .collect();
+        let array = FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 16).unwrap();
+
+        let hashes = extract_binary_hashes_from_fixed_list(&array).unwrap();
+        assert_eq!(hashes.len(), 2);
+        assert_eq!(hashes.byte_width(), 16);
+        assert_eq!(hashes.num_lanes(), 2);
+        assert_eq!(hashes.lane(0), &[rows[0].0, rows[1].0]);
+        assert_eq!(hashes.lane(1), &[rows[0].1, rows[1].1]);
+    }
+
+    #[test]
+    fn test_extract_binary_hashes_rejects_non_u64_multiple() {
+        use arrow_array::UInt8Array;
+
+        let array =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0u8; 24]), 12).unwrap();
+        let err = extract_binary_hashes_from_fixed_list(&array).unwrap_err();
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+    }
+
+    #[test]
+    fn test_binary_hash_values_rejects_width_not_divisible_by_8() {
+        let err = BinaryHashValues::try_new(Vec::new(), 0, 12).unwrap_err();
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_128() {
+        let hashes = BinaryHashValues::try_new(
+            vec![
+                0,
+                0,
+                1,
+                u64::MAX, // lane 0
+                0,
+                1,
+                1,
+                u64::MAX, // lane 1
+            ],
+            4,
+            16,
+        )
+        .unwrap();
+
+        let seq = pairwise_hamming_distance_binary(&hashes, None, Some(1));
+        let par = pairwise_hamming_distance_binary_parallel(&hashes, None, Some(1));
+        let expected = vec![(0, 1, 1), (1, 2, 1)];
+        assert_eq!(result_to_sorted_vec(&seq), expected);
+        assert_eq!(result_to_sorted_vec(&par), expected);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_parallel_128_matches_sequential() {
+        let rows: Vec<(u64, u64)> = (0..80)
+            .flat_map(|i| {
+                let lo = (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+                let hi = !lo.rotate_left(17);
+                [(lo, hi), (lo ^ 1, hi)]
+            })
+            .collect();
+        let mut lane_values = Vec::with_capacity(rows.len() * 2);
+        lane_values.extend(rows.iter().map(|(lo, _)| *lo));
+        lane_values.extend(rows.iter().map(|(_, hi)| *hi));
+        let hashes = BinaryHashValues::try_new(lane_values, rows.len(), 16).unwrap();
+        let row_ids: Vec<u64> = (0..rows.len()).map(|i| 10_000 + i as u64).collect();
+
+        let seq = pairwise_hamming_distance_binary(&hashes, Some(&row_ids), Some(1));
+        let par = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(1));
+
+        assert_eq!(result_to_sorted_vec(&par), result_to_sorted_vec(&seq));
+        assert!(par.len() >= 80);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_32_with_row_ids() {
+        let hashes = BinaryHashValues::try_new(
+            vec![
+                0,
+                0,
+                1, // lane 0
+                0,
+                1,
+                1, // lane 1
+                7,
+                7,
+                7, // lane 2
+                u64::MAX,
+                u64::MAX,
+                u64::MAX - 1, // lane 3
+            ],
+            3,
+            32,
+        )
+        .unwrap();
+        let row_ids = [10, 20, 30];
+
+        let result = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(2));
+        assert_eq!(
+            result_to_sorted_vec(&result),
+            vec![(10, 20, 1), (20, 30, 2)]
+        );
+    }
+
     #[test]
     fn test_pairwise_correctness_small() {
         // Deterministic hashes with known distances
diff --git a/rust/lance-linalg/src/distance/l2.rs b/rust/lance-linalg/src/distance/l2.rs
index c47aedd749f..7ee2f4bc537 100644
--- a/rust/lance-linalg/src/distance/l2.rs
+++ b/rust/lance-linalg/src/distance/l2.rs
@@ -20,18 +20,41 @@ use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
 use lance_core::assume_eq;
 use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
+// Named tiers are only matched on x86_64, or by the fp16 kernels on the other
+// architectures; without either, nothing below names a `SimdSupport` variant.
+#[cfg(any(feature = "fp16kernels", target_arch = "x86_64"))]
 use lance_core::utils::cpu::SimdSupport;
 use num_traits::{AsPrimitive, Num};
 
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+use crate::distance::BatchIter;
+
 /// Calculate the L2 distance between two vectors.
 ///
 pub trait L2: Num {
     /// Calculate the L2 distance between two vectors.
     fn l2(x: &[Self], y: &[Self]) -> f32;
 
-    fn l2_batch(x: &[Self], y: &[Self], dimension: usize) -> impl Iterator<Item = f32> {
-        y.chunks_exact(dimension).map(|v| Self::l2(x, v))
+    /// L2 distance from `x` to each `dimension`-sized vector in `y`.
+    ///
+    /// The default calls [`L2::l2`] per vector. `f32` overrides it so the SIMD
+    /// tier is chosen once for the whole batch instead of once per vector —
+    /// on a build whose baseline already implies AVX2, per-vector dispatch
+    /// costs more than the kernel it selects.
+    ///
+    /// Returns `impl Iterator` rather than a trait object: the k-means
+    /// assignment loop drives this one element at a time, so a
+    /// `Box<dyn Iterator>` would cost a virtual call per element and an
+    /// allocation per batch.
+    fn l2_batch<'a>(
+        x: &'a [Self],
+        y: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = f32> + 'a {
+        y.chunks_exact(dimension).map(move |v| Self::l2(x, v))
     }
 }
 
@@ -52,7 +75,6 @@ pub fn l2<T: L2>(from: &[T], to: &[T]) -> f32 {
 pub fn l2_f32(x: &[f32], y: &[f32]) -> f32 {
     #[cfg(target_arch = "x86_64")]
     {
-        use lance_core::utils::cpu::SimdSupport;
         if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) {
             // SAFETY: guarded by the runtime AVX-512 detection above.
             return unsafe { l2_f32_avx512(x, y) };
@@ -191,6 +213,9 @@ impl L2 for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::l2_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => l2_scalar::<Self, f32, 16>(x, y),
         }
     }
@@ -244,6 +269,9 @@ impl L2 for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::l2_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => l2_scalar::<Self, f32, 16>(x, y),
         }
     }
@@ -252,12 +280,114 @@ impl L2 for f16 {
 impl L2 for f32 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
-        // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32))
-        // See https://github.com/lance-format/lance/pull/2450.
-        l2_scalar::<Self, Self, 16>(x, y)
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable
+        // scalar fallback.
+        l2_f32_dispatched(x, y)
+    }
+
+    fn l2_batch<'a>(
+        x: &'a [Self],
+        y: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = Self> + 'a {
+        // Exactly one arm compiles; see `Dot::dot_batch` for f32.
+        // See `Dot::dot_batch` for f32.
+        #[cfg(all(
+            target_arch = "x86_64",
+            target_feature = "avx2",
+            target_feature = "fma"
+        ))]
+        {
+            // `l2_scalar::<_, _, 16>` chunks the vector by 16 lanes. At or below
+            // that width the chunking degenerates to its scalar remainder loop
+            // and vectorizes nothing, so the explicit AVX kernel is worth ~40%.
+            // Above it the autovectorizer already does well and the 8-wide
+            // kernel can lose, so keep the exact kernel the pre-dispatch code
+            // used and stay non-regressing by construction.
+            //
+            // SAFETY: the build baseline enables avx2+fma, which imply avx+fma,
+            // so the kernel's `#[target_feature]` contract is met statically.
+            let narrow = dimension <= 16;
+            y.chunks_exact(dimension).map(move |v| {
+                if narrow {
+                    unsafe { x86::l2_f32_avx_fma(x, v) }
+                } else {
+                    l2_f32_scalar(x, v)
+                }
+            })
+        }
+        #[cfg(all(
+            target_arch = "x86_64",
+            not(all(target_feature = "avx2", target_feature = "fma"))
+        ))]
+        {
+            l2_batch_f32_runtime_dispatch(x, y, dimension)
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            y.chunks_exact(dimension).map(move |v| Self::l2(x, v))
+        }
+    }
+}
+
+/// Sub-AVX2 builds: pick a `#[target_feature]` kernel once for the batch.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+#[inline]
+fn l2_batch_f32_runtime_dispatch<'a>(
+    x: &'a [f32],
+    y: &'a [f32],
+    dimension: usize,
+) -> impl Iterator<Item = f32> + 'a {
+    // SAFETY: each kernel is entered only under its matching runtime detection.
+    match *SIMD_SUPPORT {
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx512(x, y, dimension) }.into_iter())
+        }
+        SimdSupport::Avx2 | SimdSupport::AvxFma => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx_fma(x, y, dimension) }.into_iter())
+        }
+        SimdSupport::Avx => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx(x, y, dimension) }.into_iter())
+        }
+        _ => BatchIter::Lazy(y.chunks_exact(dimension).map(move |v| l2_f32_scalar(x, v))),
     }
 }
 
+/// L2 distance for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn l2_f32_dispatched(x: &[f32], y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f32_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f32_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::l2_f32_avx(x, y) },
+            _ => l2_f32_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        l2_f32_scalar(x, y)
+    }
+}
+
+/// Portable scalar L2 distance for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn l2_f32_scalar(x: &[f32], y: &[f32]) -> f32 {
+    // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32))
+    // See https://github.com/lance-format/lance/pull/2450.
+    l2_scalar::<f32, f32, 16>(x, y)
+}
+
 impl L2 for f64 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
@@ -265,9 +395,277 @@ impl L2 for f64 {
     }
 }
 
-/// Explicit SIMD L2 distance for f64.
+/// L2 distance for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX.
 #[inline]
 fn l2_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f64_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f64_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::l2_f64_avx(x, y) },
+            _ => l2_f64_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        l2_f64_simd_other(x, y)
+    }
+}
+
+/// Portable scalar L2 distance for f64. Used as the x86_64 fallback when no
+/// AVX2 is detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn l2_f64_scalar(x: &[f64], y: &[f64]) -> f32 {
+    x.iter()
+        .zip(y.iter())
+        .map(|(&a, &b)| {
+            let diff = a - b;
+            diff * diff
+        })
+        .sum::<f64>() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// L2 distance from `x` to every `dimension`-sized vector in `batch`, with
+    /// the AVX-512 tier entered once for the whole batch rather than once per
+    /// vector.
+    ///
+    /// # Safety
+    /// The host must support AVX-512F.
+    ///
+    /// Only compiled for builds whose baseline is below avx2+fma; at or above
+    /// that baseline `l2_batch` inlines the kernel directly and never runtime-
+    /// dispatches, so this wrapper would be dead code (see `l2_batch`).
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn l2_batch_f32_avx512(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx512(x, y) })
+            .collect()
+    }
+
+    /// As [`l2_batch_f32_avx512`], for the AVX+FMA and AVX2 tiers.
+    ///
+    /// # Safety
+    /// The host must support AVX and FMA.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn l2_batch_f32_avx_fma(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx_fma(x, y) })
+            .collect()
+    }
+
+    /// As [`l2_batch_f32_avx512`], for the AVX-without-FMA tier.
+    ///
+    /// # Safety
+    /// The host must support AVX.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn l2_batch_f32_avx(x: &[f32], batch: &[f32], dimension: usize) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx(x, y) })
+            .collect()
+    }
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vsubpd` + `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn l2_f64_avx512(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm512_loadu_pd(x.as_ptr().add(i));
+            let b = _mm512_loadu_pd(y.as_ptr().add(i));
+            let diff = _mm512_sub_pd(a, b);
+            acc = _mm512_fmadd_pd(diff, diff, acc);
+        }
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (_mm512_reduce_add_pd(acc) + tail) as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn l2_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc8.multiply_add(diff, diff);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc4.multiply_add(diff, diff);
+        }
+
+        let tail: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): squared diff via `_mm256_mul_pd` + `_mm256_add_pd` for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn l2_f64_avx(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let a = _mm256_loadu_pd(x.as_ptr().add(i));
+            let b = _mm256_loadu_pd(y.as_ptr().add(i));
+            let diff = _mm256_sub_pd(a, b);
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(diff, diff));
+        }
+
+        // Horizontal sum of __m256d -> f64.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (acc_sum + tail) as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vsubps` + `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn l2_f32_avx512(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let a = _mm512_loadu_ps(x.as_ptr().add(i));
+            let b = _mm512_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm512_sub_ps(a, b);
+            acc = _mm512_fmadd_ps(diff, diff, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        _mm512_reduce_add_ps(acc) + tail
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn l2_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm256_sub_ps(a, b);
+            acc = _mm256_fmadd_ps(diff, diff, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+
+    /// AVX-only path for f32 (no FMA): squared diff via `_mm256_mul_ps` + `_mm256_add_ps` for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn l2_f32_avx(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm256_sub_ps(a, b);
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(diff, diff));
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn l2_f64_simd_other(x: &[f64], y: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -671,6 +1069,136 @@ mod tests {
         fn test_l2_distance_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){
             do_l2_test(&x, &y)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `l2_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            let scalar = l2_f64_scalar(&x, &y);
+            let simd = l2_f64_simd(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6));
+        }
+
+        /// Parity check for `l2_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar L2 path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep
+        /// this test architecture-agnostic (the x86_64-only `l2_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_l2_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            let scalar = x
+                .iter()
+                .zip(y.iter())
+                .map(|(&a, &b)| ((a as f64) - (b as f64)).powi(2))
+                .sum::<f64>() as f32;
+            let simd = <f32 as L2>::l2(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx512 = unsafe { x86::l2_f64_avx512(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6));
+        }
+
+        /// AVX + FMA-direct parity for the f64 L2 kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx_fma = unsafe { x86::l2_f64_avx_fma(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6));
+        }
+
+        /// AVX-only-direct parity for the f64 L2 kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx = unsafe { x86::l2_f64_avx(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6));
+        }
+
+        /// AVX-512-direct parity for f32: explicitly compares the scalar
+        /// fallback against the native f32 AVX-512 inner kernel on
+        /// AVX-512F-capable hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx512 = unsafe { x86::l2_f32_avx512(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3));
+        }
+
+        /// AVX + FMA-direct parity for the f32 L2 kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx_fma = unsafe { x86::l2_f32_avx_fma(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3));
+        }
+
+        /// AVX-only-direct parity for the f32 L2 kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx = unsafe { x86::l2_f32_avx(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3));
+        }
     }
 
     #[test]
@@ -787,4 +1315,100 @@ mod tests {
         assert_relative_eq!(d1[0], 0.0); // q1 == target[0]
         assert_relative_eq!(d2[1], 0.0); // q2 == target[1]
     }
+
+    /// `l2_batch` must agree with the per-vector `l2` it replaced, on every
+    /// build: the AVX2-baseline path, the hoisted-dispatch path, and the
+    /// portable fallback all funnel through here.
+    #[rstest::rstest]
+    #[case::dim_8(8)]
+    #[case::dim_16(16)]
+    #[case::dim_32(32)]
+    #[case::dim_1024(1024)]
+    fn test_l2_batch_f32_matches_per_vector_l2(#[case] dimension: usize) {
+        let num_vectors = 5;
+        let x: Vec<f32> = (0..dimension)
+            .map(|i| ((i % 13) as f32) * 0.25 + 1.0)
+            .collect();
+        let batch: Vec<f32> = (0..dimension * num_vectors)
+            .map(|i| ((i % 11) as f32) * 0.5 - 2.0)
+            .collect();
+
+        let got: Vec<f32> = f32::l2_batch(&x, &batch, dimension).collect();
+        let want: Vec<f32> = batch
+            .chunks_exact(dimension)
+            .map(|y| f32::l2(&x, y))
+            .collect();
+
+        assert_eq!(got.len(), num_vectors);
+        for (g, w) in got.iter().zip(want.iter()) {
+            assert!(
+                approx::relative_eq!(g, w, epsilon = 1e-4),
+                "dim {dimension}: batch {g} != per-vector {w}"
+            );
+        }
+    }
+
+    /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2
+    /// builds or AVX-512 hosts, so call them directly to cover them.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    fn check_l2_batch_kernel(kernel: unsafe fn(&[f32], &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let num_vectors = 3;
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let want = l2_scalar::<f32, f32, 16>(&x, chunk);
+                assert!(
+                    approx::relative_eq!(g, want, epsilon = 1e-4),
+                    "dim {dimension}: kernel {g} != scalar {want}"
+                );
+            }
+        }
+    }
+
+    // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds
+    // (see `x86::l2_batch_f32_avx512`), so gate their tests the same way.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx_fma_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx_fma);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx512);
+    }
 }
diff --git a/rust/lance-linalg/src/distance/l2_u8.rs b/rust/lance-linalg/src/distance/l2_u8.rs
index 1b111f91338..5fbf8a3af55 100644
--- a/rust/lance-linalg/src/distance/l2_u8.rs
+++ b/rust/lance-linalg/src/distance/l2_u8.rs
@@ -143,6 +143,10 @@ fn select_backend() -> L2U8Fn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::l2_u8_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // AVX2 integer ops (`vpsubusb` / `vpmaddwd`) which neither AVX nor
+        // AVX+FMA provides.
     }
 
     l2_u8_scalar
diff --git a/rust/lance-linalg/src/distance/norm_l2.rs b/rust/lance-linalg/src/distance/norm_l2.rs
index b1daf85ab3b..8d126d6ed57 100644
--- a/rust/lance-linalg/src/distance/norm_l2.rs
+++ b/rust/lance-linalg/src/distance/norm_l2.rs
@@ -9,9 +9,7 @@ use arrow_array::types::{Float16Type, Float32Type, Float64Type};
 use arrow_schema::DataType;
 use half::{bf16, f16};
 #[allow(unused_imports)]
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 use num_traits::{AsPrimitive, Float, Num};
 
 /// L2 normalization
@@ -75,6 +73,9 @@ impl Normalize for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::norm_l2_f16_lsx(vector.as_ptr(), vector.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => norm_l2_impl::<Self, f32, 32>(vector),
         }
     }
@@ -126,6 +127,9 @@ impl Normalize for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::norm_l2_bf16_lsx(vector.as_ptr(), vector.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => norm_l2_impl::<Self, f32, 32>(vector),
         }
     }
@@ -134,10 +138,40 @@ impl Normalize for bf16 {
 impl Normalize for f32 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
-        norm_l2_impl::<Self, Self, 16>(vector)
+        norm_l2_f32_dispatched(vector)
+    }
+}
+
+/// L2 norm for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn norm_l2_f32_dispatched(vector: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                x86::norm_l2_f32_avx512(vector)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f32_avx_fma(vector) },
+            SimdSupport::Avx => unsafe { x86::norm_l2_f32_avx(vector) },
+            _ => norm_l2_f32_scalar(vector),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        norm_l2_f32_scalar(vector)
     }
 }
 
+/// Portable scalar L2 norm for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn norm_l2_f32_scalar(vector: &[f32]) -> f32 {
+    norm_l2_impl::<f32, f32, 16>(vector)
+}
+
 impl Normalize for f64 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
@@ -145,11 +179,166 @@ impl Normalize for f64 {
     }
 }
 
-/// Explicit SIMD implementation of L2 norm for f64.
+/// L2 norm for f64. Runtime-dispatched to the best available backend.
 ///
-/// Two-level unrolling: f64x8 main loop, f64x4 remainder, scalar tail.
+/// On x86_64, dispatches via `SIMD_SUPPORT` to a native AVX-512 inner kernel
+/// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4), an AVX2 + FMA kernel
+/// (Haswell+), an AVX + FMA kernel (AMD Piledriver / Steamroller), an
+/// AVX-only kernel (Intel Sandy Bridge / Ivy Bridge), or a portable scalar
+/// fallback. The per-tier inner functions each carry their own
+/// `#[target_feature]` so they stay correct under any compile baseline.
+/// On aarch64 and loongarch64, the SIMD primitives in `crate::simd::f64`
+/// are unconditionally backed by NEON / LSX-LASX respectively, so no
+/// runtime gate is required.
 #[inline]
 pub fn norm_l2_f64_simd(vector: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                x86::norm_l2_f64_avx512(vector)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f64_avx_fma(vector) },
+            SimdSupport::Avx => unsafe { x86::norm_l2_f64_avx(vector) },
+            _ => norm_l2_f64_scalar(vector),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        norm_l2_f64_simd_other(vector)
+    }
+}
+
+/// Portable scalar L2 norm. Used as the x86_64 fallback when no AVX2 is
+/// detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn norm_l2_f64_scalar(vector: &[f64]) -> f32 {
+    vector.iter().map(|v| v * v).sum::<f64>().sqrt() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn norm_l2_f64_avx512(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm512_loadu_pd(vector.as_ptr().add(i));
+            acc = _mm512_fmadd_pd(v, v, acc);
+        }
+
+        let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (_mm512_reduce_add_pd(acc) + tail).sqrt() as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn norm_l2_f64_avx_fma(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = f64x8::load_unaligned(vector.as_ptr().add(i));
+            acc8.multiply_add(v, v);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let v = f64x4::load_unaligned(vector.as_ptr().add(i));
+            acc4.multiply_add(v, v);
+        }
+
+        let tail: f64 = vector[aligned_len..].iter().map(|&v| v * v).sum();
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail).sqrt() as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn norm_l2_f64_avx(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let v = _mm256_loadu_pd(vector.as_ptr().add(i));
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(v, v));
+        }
+
+        // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (acc_sum + tail).sqrt() as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn norm_l2_f32_avx512(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let v = _mm512_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(v, v, acc);
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (_mm512_reduce_add_ps(acc) + tail).sqrt()
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn norm_l2_f32_avx_fma(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm256_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm256_fmadd_ps(v, v, acc);
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (hsum256_ps(acc) + tail).sqrt()
+    }
+
+    /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn norm_l2_f32_avx(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm256_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v));
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (hsum256_ps(acc) + tail).sqrt()
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn norm_l2_f64_simd_other(vector: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -291,5 +480,133 @@ mod tests {
         fn test_l2_norm_f64(data in prop::collection::vec(arbitrary_f64(), 4..4048)){
             do_norm_l2_test(&data)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `norm_l2_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_simd_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            let scalar = norm_l2_f64_scalar(&data);
+            let simd = norm_l2_f64_simd(&data);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6));
+        }
+
+        /// Parity check for `norm_l2_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar L2-norm path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep this
+        /// test architecture-agnostic (the x86_64-only `norm_l2_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_l2_norm_f32_scalar_simd_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            let scalar = data.iter().map(|&v| (v as f64).powi(2)).sum::<f64>().sqrt() as f32;
+            let simd = <f32 as Normalize>::norm_l2(&data);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F so the test stays portable; CI runners with
+        /// AVX-512F exercise the `_mm512_*` path.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx512_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx512 = unsafe { x86::norm_l2_f64_avx512(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6));
+        }
+
+        /// AVX + FMA-direct parity for the f64 L2-norm kernel. Covers the
+        /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on
+        /// hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx_fma_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx_fma = unsafe { x86::norm_l2_f64_avx_fma(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6));
+        }
+
+        /// AVX-only-direct parity for the f64 L2-norm kernel. Covers the
+        /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA).
+        /// Early-returns on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx = unsafe { x86::norm_l2_f64_avx(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6));
+        }
+
+        /// AVX-512-direct parity for the f32 L2-norm kernel. Explicitly
+        /// compares the scalar fallback against the native AVX-512 inner
+        /// kernel on AVX-512F-capable hosts. Early-returns on hosts without
+        /// AVX-512F; CI runners with AVX-512F exercise the `_mm512_*` path.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx512_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx512 = unsafe { x86::norm_l2_f32_avx512(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3));
+        }
+
+        /// AVX + FMA-direct parity for the f32 L2-norm kernel. Covers the
+        /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on
+        /// hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx_fma_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx_fma = unsafe { x86::norm_l2_f32_avx_fma(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3));
+        }
+
+        /// AVX-only-direct parity for the f32 L2-norm kernel. Covers the
+        /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA).
+        /// Early-returns on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx = unsafe { x86::norm_l2_f32_avx(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3));
+        }
     }
 }
diff --git a/rust/lance-linalg/src/simd.rs b/rust/lance-linalg/src/simd.rs
index 91dc1c6959d..6722eafd768 100644
--- a/rust/lance-linalg/src/simd.rs
+++ b/rust/lance-linalg/src/simd.rs
@@ -19,6 +19,8 @@ pub mod f32;
 pub mod f64;
 pub mod i32;
 pub mod u8;
+#[cfg(target_arch = "x86_64")]
+pub(crate) mod x86;
 
 use num_traits::{Float, Num};
 use u8::u8x16;
diff --git a/rust/lance-linalg/src/simd/dist_table.rs b/rust/lance-linalg/src/simd/dist_table.rs
index 626c1581b15..e337953ec10 100644
--- a/rust/lance-linalg/src/simd/dist_table.rs
+++ b/rust/lance-linalg/src/simd/dist_table.rs
@@ -73,6 +73,10 @@ pub fn sum_4bit_dist_table(
                 )
             }
         },
+        // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+        // the AVX2 inner uses `_mm256_shuffle_epi8` / `_mm256_and_si256` /
+        // `_mm256_srli_epi16` / `_mm256_add_epi16` integer ops which
+        // neither AVX nor AVX+FMA provides. Scalar is the correct route.
         _ => sum_4bit_dist_table_scalar(code_len, codes, dist_table, dists),
     }
 }
diff --git a/rust/lance-linalg/src/simd/f32.rs b/rust/lance-linalg/src/simd/f32.rs
index 78042997121..4ce7f64706d 100644
--- a/rust/lance-linalg/src/simd/f32.rs
+++ b/rust/lance-linalg/src/simd/f32.rs
@@ -46,14 +46,42 @@ impl std::fmt::Debug for f32x8 {
 }
 
 impl f32x8 {
+    /// Gather 8 f32 values from `slice` at the offsets in `indices`.
+    ///
+    /// On x86_64 this uses the AVX2 `vgatherdps` instruction when the host
+    /// supports it (gated at runtime via `is_x86_feature_detected!`). On
+    /// other architectures (and on x86_64 hosts without AVX2) the function
+    /// falls back to a per-index scalar load followed by `Self::from(&out)`,
+    /// which goes through `load_unaligned` (NEON / LASX / `_mm256_loadu_ps`
+    /// depending on platform). Per-tier macro stamping (e.g., the
+    /// `multiversion` crate) was considered but doesn't fit here: the function
+    /// returns `Self` and `_mm256_i32gather_ps::<4>` requires the const-generic
+    /// stride to be a compile-time literal — neither composes with the macro.
+    ///
+    /// # Panics
+    ///
+    /// If any index is negative or lands outside `slice`.
     #[inline]
     pub fn gather(slice: &[f32], indices: &[i32; 8]) -> Self {
-        #[cfg(target_arch = "x86_64")]
-        unsafe {
-            use super::i32::i32x8;
+        // Every backend below reads without bounds checking: `vgatherdps` does
+        // none, and the NEON / LASX arms offset a raw pointer. Check once here
+        // so an out-of-range index panics on every host rather than reading out
+        // of bounds on some and panicking on others.
+        for &i in indices {
+            assert!(
+                (i as usize) < slice.len(),
+                "gather index {i} is out of bounds for a slice of length {}",
+                slice.len()
+            );
+        }
 
-            let idx = i32x8::from(indices);
-            Self(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0))
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("avx2") {
+                unsafe { gather_avx2(slice, indices) }
+            } else {
+                gather_scalar_x86(slice, indices)
+            }
         }
 
         #[cfg(target_arch = "aarch64")]
@@ -94,6 +122,30 @@ impl f32x8 {
     }
 }
 
+/// AVX2 gather. Caller must ensure the host supports AVX2 (gated by
+/// the `is_x86_feature_detected!("avx2")` check in `f32x8::gather`).
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn gather_avx2(slice: &[f32], indices: &[i32; 8]) -> f32x8 {
+    use super::i32::i32x8;
+
+    let idx = i32x8::from(indices);
+    f32x8(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0))
+}
+
+/// Portable scalar gather for x86_64 hosts without AVX2.
+///
+/// Indexes the slice rather than offsetting a raw pointer: this is the slow
+/// path already, so an out-of-range index should panic instead of reading out
+/// of bounds.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn gather_scalar_x86(slice: &[f32], indices: &[i32; 8]) -> f32x8 {
+    let values = indices.map(|i| slice[i as usize]);
+    // SAFETY: `values` is eight contiguous, initialized `f32`.
+    unsafe { f32x8::load_unaligned(values.as_ptr()) }
+}
+
 impl From<&[f32]> for f32x8 {
     fn from(value: &[f32]) -> Self {
         unsafe { Self::load_unaligned(value.as_ptr()) }
@@ -439,15 +491,18 @@ impl Mul for f32x8 {
     }
 }
 
-/// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible.
+/// 16 of 32-bit `f32` values. Stored as a pair of 256-bit AVX vectors on
+/// x86_64. Originally there was a sibling AVX-512 variant gated on
+/// `target_feature = "avx512f"`, but no project CI configuration enables
+/// `+avx512f` globally (and one of the avx512 arms still contained `todo!()`),
+/// so the variant was dead code. Removed in the runtime-SIMD-dispatch
+/// retrofit; per-tier dispatch happens in the kernel functions in
+/// `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier
+/// `#[target_feature(enable = "...")]` inner functions.
 #[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+#[cfg(target_arch = "x86_64")]
 #[derive(Clone, Copy)]
 pub struct f32x16(__m256, __m256);
-#[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-#[derive(Clone, Copy)]
-pub struct f32x16(__m512);
 
 /// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible.
 #[allow(non_camel_case_types)]
@@ -486,11 +541,7 @@ impl<'a> From<&'a [f32; 16]> for f32x16 {
 impl SIMD<f32, 16> for f32x16 {
     #[inline]
     fn splat(val: f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_set1_ps(val))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_set1_ps(val), _mm256_set1_ps(val))
         }
@@ -514,11 +565,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn zeros() -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_setzero_ps())
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_setzero_ps(), _mm256_setzero_ps())
         }
@@ -534,14 +581,10 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn load(ptr: *const f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_load_ps(ptr), _mm256_load_ps(ptr.add(8)))
         }
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_load_ps(ptr))
-        }
         #[cfg(target_arch = "aarch64")]
         {
             Self::load_unaligned(ptr)
@@ -557,14 +600,10 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn load_unaligned(ptr: *const f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_loadu_ps(ptr), _mm256_loadu_ps(ptr.add(8)))
         }
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_loadu_ps(ptr))
-        }
         #[cfg(target_arch = "aarch64")]
         {
             Self(vld1q_f32_x4(ptr))
@@ -580,11 +619,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn store(&self, ptr: *mut f32) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_store_ps(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_store_ps(ptr, self.0);
             _mm256_store_ps(ptr.add(8), self.1);
@@ -602,11 +637,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn store_unaligned(&self, ptr: *mut f32) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_storeu_ps(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_storeu_ps(ptr, self.0);
             _mm256_storeu_ps(ptr.add(8), self.1);
@@ -622,12 +653,9 @@ impl SIMD<f32, 16> for f32x16 {
         }
     }
 
+    #[inline]
     fn reduce_sum(&self) -> f32 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_add_ps(0xFFFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let mut sum = _mm256_add_ps(self.0, self.1);
             // Shift and add vector, until only 1 value left.
@@ -657,11 +685,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn reduce_min(&self) -> f32 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_min_ps(0xFFFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let mut m1 = _mm256_min_ps(self.0, self.1);
             let mut m2 = _mm256_permute2f128_ps(m1, m1, 1);
@@ -695,11 +719,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn min(&self, rhs: &Self) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_min_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_min_ps(self.0, rhs.0), _mm256_min_ps(self.1, rhs.1))
         }
@@ -718,21 +738,15 @@ impl SIMD<f32, 16> for f32x16 {
         }
     }
 
+    #[inline]
     fn find(&self, val: f32) -> Option<i32> {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            // let tgt = _mm512_set1_ps(val);
-            // let mask = _mm512_cmpeq_ps_mask(self.0, tgt);
-            // if mask != 0 {
-            //     return Some(mask.trailing_zeros() as i32);
-            // }
-            todo!()
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
-            // _mm256_cmpeq_ps_mask requires "avx512l".
+            // _mm256_cmpeq_ps_mask requires AVX-512 (avx512f); use a scalar scan here
+            // since we only require AVX2.
+            let arr = self.as_array();
             for i in 0..16 {
-                if self.as_array().get_unchecked(i) == &val {
+                if arr.get_unchecked(i) == &val {
                     return Some(i as i32);
                 }
             }
@@ -774,11 +788,7 @@ impl SIMD<f32, 16> for f32x16 {
 impl FloatSimd<f32, 16> for f32x16 {
     #[inline]
     fn multiply_add(&mut self, a: Self, b: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_fmadd_ps(a.0, b.0, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_fmadd_ps(a.0, b.0, self.0);
             self.1 = _mm256_fmadd_ps(a.1, b.1, self.1);
@@ -803,11 +813,7 @@ impl Add for f32x16 {
 
     #[inline]
     fn add(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_add_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_add_ps(self.0, rhs.0), _mm256_add_ps(self.1, rhs.1))
         }
@@ -830,11 +836,7 @@ impl Add for f32x16 {
 impl AddAssign for f32x16 {
     #[inline]
     fn add_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_add_ps(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_add_ps(self.0, rhs.0);
             self.1 = _mm256_add_ps(self.1, rhs.1);
@@ -859,11 +861,7 @@ impl Mul for f32x16 {
 
     #[inline]
     fn mul(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_mul_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_mul_ps(self.0, rhs.0), _mm256_mul_ps(self.1, rhs.1))
         }
@@ -888,11 +886,7 @@ impl Sub for f32x16 {
 
     #[inline]
     fn sub(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_sub_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_sub_ps(self.0, rhs.0), _mm256_sub_ps(self.1, rhs.1))
         }
@@ -915,11 +909,7 @@ impl Sub for f32x16 {
 impl SubAssign for f32x16 {
     #[inline]
     fn sub_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_sub_ps(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_sub_ps(self.0, rhs.0);
             self.1 = _mm256_sub_ps(self.1, rhs.1);
@@ -943,9 +933,17 @@ impl SubAssign for f32x16 {
 mod tests {
 
     use super::*;
+    use rstest::rstest;
 
     #[test]
     fn test_basic_ops() {
+        // Load / store / arithmetic on `f32x8` lower to AVX intrinsics, and
+        // `multiply_add` lowers to `_mm256_fmadd_ps`, which needs FMA. Both
+        // are present from the AvxFma tier up.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
         let b = (10..18).map(|f| f as f32).collect::<Vec<_>>();
 
@@ -983,6 +981,11 @@ mod tests {
 
     #[test]
     fn test_f32x8_cmp_ops() {
+        // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0];
         let b = [2.0_f32, 1.0, 4.0, 5.0, 9.0, 5.0, 6.0, 2.0];
         let c = [2.0_f32, 1.0, 4.0, 5.0, 7.0, 3.0, 2.0, 1.0];
@@ -1007,6 +1010,11 @@ mod tests {
 
     #[test]
     fn test_basic_f32x16_ops() {
+        // `f32x16` is a pair of `__m256`; `multiply_add` needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = (0..16).map(|f| f as f32).collect::<Vec<_>>();
         let b = (10..26).map(|f| f as f32).collect::<Vec<_>>();
 
@@ -1041,6 +1049,11 @@ mod tests {
 
     #[test]
     fn test_f32x16_cmp_ops() {
+        // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [
             1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0, -0.5, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0,
         ];
@@ -1074,9 +1087,59 @@ mod tests {
 
     #[test]
     fn test_f32x8_gather() {
+        // `f32x8::gather` does its own runtime AVX2 detection and falls back
+        // to a scalar gather, so this test only needs whatever reading the
+        // `__m256`-backed result costs: AVX, for `reduce_sum`.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = (0..256).map(|f| f as f32).collect::<Vec<_>>();
         let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29];
         let v = f32x8::gather(&a, &idx);
         assert_eq!(v.reduce_sum(), 113.0);
     }
+
+    /// Directly exercises `gather_scalar_x86`, the per-index scalar fallback
+    /// `f32x8::gather` takes on x86_64 hosts without AVX2. Runtime AVX2 hosts
+    /// route through `gather_avx2` instead, so the fallback is otherwise never
+    /// hit under coverage. Reading the `__m256`-backed result needs AVX, so
+    /// skip on hosts without it.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_gather_scalar_x86() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let a = (0..256).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29];
+        let v = gather_scalar_x86(&a, &idx);
+        let expected = idx.map(|i| a[i as usize]);
+        assert_eq!(v.as_array(), expected);
+    }
+
+    /// An index past the end of the slice panics rather than reading out of
+    /// bounds. The bounds check fires before any AVX instruction, so this
+    /// case runs on every x86_64 host.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    #[should_panic(expected = "index out of bounds")]
+    fn test_gather_scalar_x86_rejects_out_of_range_index() {
+        let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 1, 2, 3, 4, 5, 6, 99];
+        let _ = gather_scalar_x86(&a, &idx);
+    }
+
+    /// `gather` validates before dispatching, so every backend — `vgatherdps`,
+    /// the x86 scalar fallback, and the NEON / LASX raw-pointer arms — rejects
+    /// a bad index identically instead of reading out of bounds.
+    #[rstest]
+    #[case::past_end(99)]
+    #[case::negative(-1)]
+    #[should_panic(expected = "out of bounds")]
+    fn test_gather_rejects_invalid_index(#[case] bad_index: i32) {
+        let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 1, 2, 3, 4, 5, 6, bad_index];
+        let _ = f32x8::gather(&a, &idx);
+    }
 }
diff --git a/rust/lance-linalg/src/simd/f64.rs b/rust/lance-linalg/src/simd/f64.rs
index 32c0d389e5b..1276f54da56 100644
--- a/rust/lance-linalg/src/simd/f64.rs
+++ b/rust/lance-linalg/src/simd/f64.rs
@@ -355,14 +355,15 @@ impl Mul for f64x4 {
 // f64x8: 8 × f64 values (512-bit SIMD or 2 × 256-bit)
 // ---------------------------------------------------------------------------
 
-/// 8 of 64-bit `f64` values. Uses 512-bit SIMD if possible.
+/// 8 of 64-bit `f64` values. Stored as a pair of 256-bit AVX vectors on
+/// x86_64. Originally there was a sibling AVX-512 variant gated on
+/// `target_feature = "avx512f"`, but no project CI configuration enables
+/// `+avx512f` globally, so the variant was dead code. Removed in the
+/// runtime-SIMD-dispatch retrofit; per-tier dispatch happens in the kernel
+/// functions in `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier
+/// `#[target_feature(enable = "...")]` inner functions.
 #[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-#[derive(Clone, Copy)]
-pub struct f64x8(__m512d);
-
-#[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+#[cfg(target_arch = "x86_64")]
 #[derive(Clone, Copy)]
 pub struct f64x8(__m256d, __m256d);
 
@@ -401,11 +402,7 @@ impl<'a> From<&'a [f64; 8]> for f64x8 {
 impl SIMD<f64, 8> for f64x8 {
     #[inline]
     fn splat(val: f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_set1_pd(val))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_set1_pd(val), _mm256_set1_pd(val))
         }
@@ -423,11 +420,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn zeros() -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_setzero_pd())
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_setzero_pd(), _mm256_setzero_pd())
         }
@@ -443,11 +436,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn load(ptr: *const f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_load_pd(ptr))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_load_pd(ptr), _mm256_load_pd(ptr.add(4)))
         }
@@ -466,11 +455,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn load_unaligned(ptr: *const f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_loadu_pd(ptr))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_loadu_pd(ptr), _mm256_loadu_pd(ptr.add(4)))
         }
@@ -489,11 +474,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn store(&self, ptr: *mut f64) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_store_pd(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_store_pd(ptr, self.0);
             _mm256_store_pd(ptr.add(4), self.1);
@@ -512,11 +493,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn store_unaligned(&self, ptr: *mut f64) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_storeu_pd(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_storeu_pd(ptr, self.0);
             _mm256_storeu_pd(ptr.add(4), self.1);
@@ -533,12 +510,9 @@ impl SIMD<f64, 8> for f64x8 {
         }
     }
 
+    #[inline]
     fn reduce_sum(&self) -> f64 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_add_pd(0xFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let sum = _mm256_add_pd(self.0, self.1);
             let hi = _mm256_permute2f128_pd(sum, sum, 1);
@@ -561,11 +535,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn reduce_min(&self) -> f64 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_min_pd(0xFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let m = _mm256_min_pd(self.0, self.1);
             let hi = _mm256_permute2f128_pd(m, m, 1);
@@ -592,11 +562,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn min(&self, rhs: &Self) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_min_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_min_pd(self.0, rhs.0), _mm256_min_pd(self.1, rhs.1))
         }
@@ -613,6 +579,7 @@ impl SIMD<f64, 8> for f64x8 {
         }
     }
 
+    #[inline]
     fn find(&self, val: f64) -> Option<i32> {
         unsafe {
             for i in 0..8 {
@@ -628,11 +595,7 @@ impl SIMD<f64, 8> for f64x8 {
 impl FloatSimd<f64, 8> for f64x8 {
     #[inline]
     fn multiply_add(&mut self, a: Self, b: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_fmadd_pd(a.0, b.0, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_fmadd_pd(a.0, b.0, self.0);
             self.1 = _mm256_fmadd_pd(a.1, b.1, self.1);
@@ -657,11 +620,7 @@ impl Add for f64x8 {
 
     #[inline]
     fn add(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_add_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_add_pd(self.0, rhs.0), _mm256_add_pd(self.1, rhs.1))
         }
@@ -682,11 +641,7 @@ impl Add for f64x8 {
 impl AddAssign for f64x8 {
     #[inline]
     fn add_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_add_pd(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_add_pd(self.0, rhs.0);
             self.1 = _mm256_add_pd(self.1, rhs.1);
@@ -711,11 +666,7 @@ impl Mul for f64x8 {
 
     #[inline]
     fn mul(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_mul_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_mul_pd(self.0, rhs.0), _mm256_mul_pd(self.1, rhs.1))
         }
@@ -738,11 +689,7 @@ impl Sub for f64x8 {
 
     #[inline]
     fn sub(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_sub_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_sub_pd(self.0, rhs.0), _mm256_sub_pd(self.1, rhs.1))
         }
@@ -763,11 +710,7 @@ impl Sub for f64x8 {
 impl SubAssign for f64x8 {
     #[inline]
     fn sub_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_sub_pd(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_sub_pd(self.0, rhs.0);
             self.1 = _mm256_sub_pd(self.1, rhs.1);
@@ -793,6 +736,12 @@ mod tests {
 
     #[test]
     fn test_f64x4_basic_ops() {
+        // The `f64x4` constructor / load / store / arithmetic paths all lower
+        // to AVX intrinsics on x86_64; none of them need AVX2.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f64, 2.0, 3.0, 4.0];
         let b = [5.0_f64, 6.0, 7.0, 8.0];
 
@@ -814,6 +763,11 @@ mod tests {
 
     #[test]
     fn test_f64x4_fma() {
+        // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = [1.0_f64, 2.0, 3.0, 4.0];
         let b = [2.0_f64, 3.0, 4.0, 5.0];
 
@@ -826,6 +780,11 @@ mod tests {
 
     #[test]
     fn test_f64x4_min() {
+        // `min` / `reduce_min` are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f64, 5.0, 2.0, 8.0];
         let b = [3.0_f64, 2.0, 4.0, 1.0];
         let simd_a: f64x4 = (&a).into();
@@ -838,6 +797,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_basic_ops() {
+        // `f64x8` is a pair of `__m256d`; add / reduce are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
         let b: [f64; 8] = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0];
 
@@ -856,6 +820,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_fma() {
+        // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
         let b: [f64; 8] = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0];
 
@@ -869,6 +838,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_min() {
+        // `min` / `reduce_min` are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a: [f64; 8] = [5.0, 1.0, 8.0, 3.0, 9.0, 2.0, 7.0, 4.0];
         let b: [f64; 8] = [2.0, 6.0, 3.0, 7.0, 1.0, 8.0, 4.0, 9.0];
         let simd_a: f64x8 = (&a).into();
diff --git a/rust/lance-linalg/src/simd/x86.rs b/rust/lance-linalg/src/simd/x86.rs
new file mode 100644
index 00000000000..437018669e2
--- /dev/null
+++ b/rust/lance-linalg/src/simd/x86.rs
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Reduction helpers shared by the x86_64 AVX kernels in [`crate::distance`].
+//!
+//! Each distance kernel accumulates into a 256-bit register and folds it down
+//! to a scalar once, after the main loop. The fold is identical across
+//! `cosine`, `dot`, `l2` and `norm_l2`, so it lives here instead of being
+//! copied into each kernel's private `mod x86`.
+//!
+//! The module itself is `pub(crate)`, which is what keeps these helpers off the
+//! public API; the items are `pub` rather than `pub(crate)` only because
+//! `clippy::redundant_pub_crate` fires on the narrower visibility.
+
+use std::arch::x86_64::*;
+
+/// Horizontal sum of the eight `f32` lanes of an `__m256`.
+///
+/// Folds the upper 128-bit lane into the lower one, then reduces the
+/// remaining four lanes pairwise. Uses `movehl`/`shuffle` plus scalar adds
+/// rather than two `vhaddps`, which is one fewer uop on most cores.
+///
+/// # Safety
+///
+/// The host must support AVX. Callers are `#[target_feature]`-annotated
+/// kernels that the runtime dispatcher only selects after checking.
+#[inline]
+#[target_feature(enable = "avx")]
+pub unsafe fn hsum256_ps(v: __m256) -> f32 {
+    let lo = _mm256_castps256_ps128(v);
+    let hi = _mm256_extractf128_ps(v, 1);
+    let sum128 = _mm_add_ps(lo, hi);
+    let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128));
+    // 0x55 broadcasts lane 1 into lane 0, so the scalar add below lands the
+    // last of the four partial sums.
+    let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55));
+    _mm_cvtss_f32(sum32)
+}
+
+/// Horizontal sum of the four `f64` lanes of an `__m256d`.
+///
+/// Folds the upper 128-bit lane into the lower one, then adds the remaining
+/// pair.
+///
+/// # Safety
+///
+/// The host must support AVX. Callers are `#[target_feature]`-annotated
+/// kernels that the runtime dispatcher only selects after checking.
+#[inline]
+#[target_feature(enable = "avx")]
+pub unsafe fn hsum256_pd(v: __m256d) -> f64 {
+    let lo = _mm256_castpd256_pd128(v);
+    let hi = _mm256_extractf128_pd(v, 1);
+    let sum128 = _mm_add_pd(lo, hi);
+    let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+    _mm_cvtsd_f64(sum64)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::ascending([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], 36.0)]
+    #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5, 0.0, -0.5, 1.0, 2.5], 5.0)]
+    #[case::zeros([0.0; 8], 0.0)]
+    #[case::cancelling([1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0, -4.0], 0.0)]
+    fn hsum256_ps_sums_every_lane(#[case] lanes: [f32; 8], #[case] expected: f32) {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let sum = unsafe { hsum256_ps(_mm256_loadu_ps(lanes.as_ptr())) };
+        assert_eq!(sum, expected);
+    }
+
+    #[rstest]
+    #[case::ascending([1.0, 2.0, 3.0, 4.0], 10.0)]
+    #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5], 2.0)]
+    #[case::zeros([0.0; 4], 0.0)]
+    #[case::cancelling([1.0, -1.0, 2.0, -2.0], 0.0)]
+    fn hsum256_pd_sums_every_lane(#[case] lanes: [f64; 4], #[case] expected: f64) {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let sum = unsafe { hsum256_pd(_mm256_loadu_pd(lanes.as_ptr())) };
+        assert_eq!(sum, expected);
+    }
+}
diff --git a/rust/lance-namespace-datafusion/src/catalog.rs b/rust/lance-namespace-datafusion/src/catalog.rs
index 4fe57f63c9b..ce699037ba0 100755
--- a/rust/lance-namespace-datafusion/src/catalog.rs
+++ b/rust/lance-namespace-datafusion/src/catalog.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::collections::HashSet;
 use std::sync::Arc;
 
@@ -53,10 +52,6 @@ impl LanceCatalogProviderList {
 }
 
 impl CatalogProviderList for LanceCatalogProviderList {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     /// Adds a new catalog to this catalog list.
     /// If a catalog of the same name existed before, it is replaced in the list and returned.
     fn register_catalog(
@@ -116,10 +111,6 @@ impl LanceCatalogProvider {
 }
 
 impl CatalogProvider for LanceCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema_names(&self) -> Vec<String> {
         self.schemas
             .iter()
diff --git a/rust/lance-namespace-datafusion/src/schema.rs b/rust/lance-namespace-datafusion/src/schema.rs
index 9acf30a97bf..194346001c3 100755
--- a/rust/lance-namespace-datafusion/src/schema.rs
+++ b/rust/lance-namespace-datafusion/src/schema.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -51,10 +50,6 @@ impl LanceSchemaProvider {
 
 #[async_trait]
 impl SchemaProvider for LanceSchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn table_names(&self) -> Vec<String> {
         self.tables
             .iter()
diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml
index 27b9a4bc0e2..c5d8e120740 100644
--- a/rust/lance-namespace-impls/Cargo.toml
+++ b/rust/lance-namespace-impls/Cargo.toml
@@ -91,7 +91,7 @@ rustls-pki-types = { version = "1", optional = true }
 # Azure credential vending dependencies (optional, enabled by "credential-vendor-azure" feature)
 chrono = { workspace = true, optional = true }
 hmac = { version = "0.12", optional = true }
-quick-xml = { version = "0.38", optional = true }
+quick-xml = { version = "0.40", optional = true }
 
 [dev-dependencies]
 opendal = { workspace = true, features = ["services-goosefs"] }
diff --git a/rust/lance-namespace-impls/src/credentials.rs b/rust/lance-namespace-impls/src/credentials.rs
index e841ac620f6..23f9346cfb3 100644
--- a/rust/lance-namespace-impls/src/credentials.rs
+++ b/rust/lance-namespace-impls/src/credentials.rs
@@ -223,6 +223,14 @@ pub mod aws_props {
     /// AWS credential duration in milliseconds.
     /// Default: 3600000 (1 hour). Range: 900000 (15 min) to 43200000 (12 hours).
     pub const DURATION_MILLIS: &str = "aws_duration_millis";
+
+    /// When "true", the scoped assume is performed via `AssumeRoleWithWebIdentity`
+    /// using the pod's projected service-account OIDC token
+    pub const ASSUME_VIA_POD_WEB_IDENTITY: &str = "aws_assume_via_pod_web_identity";
+
+    /// Explicit path to the pod's projected SA OIDC token file. Overrides
+    /// `AWS_WEB_IDENTITY_TOKEN_FILE` when set.
+    pub const POD_WEB_IDENTITY_TOKEN_FILE: &str = "aws_pod_web_identity_token_file";
 }
 
 /// GCP-specific property keys (short form, without prefix)
@@ -503,6 +511,43 @@ async fn create_aws_vendor(
         config = config.with_role_session_name(session_name);
     }
 
+    // Direct (non-chained) web-identity assume for the pod, when enabled. An
+    // explicit token-file path wins; otherwise, if opted in, resolve the
+    // EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`. Falling back to the chained
+    // AssumeRole path when neither is present keeps existing behavior.
+    let assume_via_pod = properties
+        .get(aws_props::ASSUME_VIA_POD_WEB_IDENTITY)
+        .map(|v| v.eq_ignore_ascii_case("true"))
+        .unwrap_or(false);
+    let pod_token_file = properties
+        .get(aws_props::POD_WEB_IDENTITY_TOKEN_FILE)
+        .cloned()
+        .or_else(|| {
+            assume_via_pod
+                .then(|| std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE").ok())
+                .flatten()
+        });
+    // Log the resolved assume path once at vendor init so a deployment can
+    // confirm at runtime which branch `assume_scoped` will take.
+    match &pod_token_file {
+        Some(path) => log::info!(
+            "AWS credential vendor (role {role_arn}): direct AssumeRoleWithWebIdentity \
+             via pod token file '{path}'"
+        ),
+        None if assume_via_pod => log::warn!(
+            "AWS credential vendor (role {role_arn}): aws_assume_via_pod_web_identity=true \
+             but no token file resolved (aws_pod_web_identity_token_file unset and \
+             AWS_WEB_IDENTITY_TOKEN_FILE not in env); falling back to chained AssumeRole"
+        ),
+        None => log::info!(
+            "AWS credential vendor (role {role_arn}): chained AssumeRole \
+             (pod web-identity not enabled)"
+        ),
+    }
+    if let Some(path) = pod_token_file {
+        config = config.with_pod_web_identity_token_file(path);
+    }
+
     let vendor = AwsCredentialVendor::new(config).await?;
     Ok(Some(Box::new(vendor)))
 }
diff --git a/rust/lance-namespace-impls/src/credentials/aws.rs b/rust/lance-namespace-impls/src/credentials/aws.rs
index 7dedaa6e108..56dc9a54c2c 100644
--- a/rust/lance-namespace-impls/src/credentials/aws.rs
+++ b/rust/lance-namespace-impls/src/credentials/aws.rs
@@ -24,6 +24,18 @@ use super::{
     redact_credential,
 };
 
+/// Render an error together with its full `source()` chain.
+fn full_error_chain(err: &dyn std::error::Error) -> String {
+    let mut out = err.to_string();
+    let mut source = err.source();
+    while let Some(cause) = source {
+        out.push_str(": ");
+        out.push_str(&cause.to_string());
+        source = cause.source();
+    }
+    out
+}
+
 /// Configuration for AWS credential vending.
 #[derive(Debug, Clone)]
 pub struct AwsCredentialVendorConfig {
@@ -60,6 +72,17 @@ pub struct AwsCredentialVendorConfig {
     /// When an API key is provided, its hash is looked up in this map.
     /// If found, the mapped permission is used instead of the default permission.
     pub api_key_hash_permissions: HashMap<String, VendedPermission>,
+
+    /// Optional path to the pod's projected service-account OIDC token file (typically the
+    /// EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`).
+    /// This is the recommended method when running in Kubernetes.
+    /// When set, the scoped assume is performed with `AssumeRoleWithWebIdentity`
+    /// using this token instead of a role-chained `AssumeRole` from the process's
+    /// ambient credentials. A web-identity assumption is *not* role chaining, so
+    /// STS honors the role's `MaxSessionDuration` (up to 12h) and the returned
+    /// expiration is accurate. When `None`, the `AssumeRole` path
+    /// is used and `duration_millis` is subject to the source role duration and may be invalid
+    pub pod_web_identity_token_file: Option<String>,
 }
 
 impl AwsCredentialVendorConfig {
@@ -74,6 +97,7 @@ impl AwsCredentialVendorConfig {
             permission: VendedPermission::default(),
             api_key_salt: None,
             api_key_hash_permissions: HashMap::new(),
+            pod_web_identity_token_file: None,
         }
     }
 
@@ -101,6 +125,14 @@ impl AwsCredentialVendorConfig {
         self
     }
 
+    /// Set the pod web-identity token file, enabling direct (non-chained)
+    /// `AssumeRoleWithWebIdentity` for the scoped assume. See
+    /// [`AwsCredentialVendorConfig::pod_web_identity_token_file`].
+    pub fn with_pod_web_identity_token_file(mut self, path: impl Into<String>) -> Self {
+        self.pod_web_identity_token_file = Some(path.into());
+        self
+    }
+
     /// Set the permission level for vended credentials.
     pub fn with_permission(mut self, permission: VendedPermission) -> Self {
         self.permission = permission;
@@ -407,7 +439,8 @@ impl AwsCredentialVendor {
                 lance_core::Error::from(NamespaceError::Internal {
                     message: format!(
                         "AssumeRoleWithWebIdentity failed for role '{}': {}",
-                        self.config.role_arn, e
+                        self.config.role_arn,
+                        full_error_chain(&e)
                     ),
                 })
             })?;
@@ -421,6 +454,95 @@ impl AwsCredentialVendor {
     }
 
     /// Vend credentials using AssumeRole with API key validation.
+    /// Perform the scoped assume for `(bucket, prefix, permission)`, attaching the
+    /// per-table session policy.
+    ///
+    /// When [`AwsCredentialVendorConfig::pod_web_identity_token_file`] is set this
+    /// uses `AssumeRoleWithWebIdentity` with the pod's projected SA OIDC token -- a
+    /// *direct*, non-chained assumption that honors the role's `MaxSessionDuration`
+    /// (so `expires_at_millis` is accurate). Otherwise it falls back to the legacy
+    /// role-chained `AssumeRole` from ambient credentials (STS-capped at 1h).
+    ///
+    /// `external_id` is only applied on the chained `AssumeRole` path;
+    /// `AssumeRoleWithWebIdentity` has no external-id parameter (the OIDC
+    /// `sub`/`aud` trust condition is the binding instead).
+    async fn assume_scoped(
+        &self,
+        bucket: &str,
+        prefix: &str,
+        permission: VendedPermission,
+        session_name: &str,
+        external_id: Option<&str>,
+    ) -> Result<VendedCredentials> {
+        let policy = Self::build_policy(bucket, prefix, permission);
+        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
+
+        let credentials = if let Some(token_file) = &self.config.pod_web_identity_token_file {
+            // DIRECT (non-chained): AssumeRoleWithWebIdentity with the pod's SA
+            // OIDC token. Re-read the file every vend -- kubelet rotates it.
+            let token = tokio::fs::read_to_string(token_file).await.map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "failed to read pod web identity token '{}': {}",
+                        token_file, e
+                    ),
+                })
+            })?;
+            debug!(
+                "AWS AssumeRoleWithWebIdentity (pod): role={}, session={}, permission={}",
+                self.config.role_arn, session_name, permission
+            );
+            let response = self
+                .sts_client
+                .assume_role_with_web_identity()
+                .role_arn(&self.config.role_arn)
+                .web_identity_token(token.trim())
+                .role_session_name(session_name)
+                .policy(&policy)
+                .duration_seconds(duration_secs)
+                .send()
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "AssumeRoleWithWebIdentity (pod) failed for role '{}': {}",
+                            self.config.role_arn,
+                            full_error_chain(&e)
+                        ),
+                    })
+                })?;
+            response.credentials().cloned()
+        } else {
+            // LEGACY chained path: AssumeRole from ambient credentials (1h cap).
+            debug!(
+                "AWS AssumeRole (chained): role={}, session={}, permission={}",
+                self.config.role_arn, session_name, permission
+            );
+            let mut request = self
+                .sts_client
+                .assume_role()
+                .role_arn(&self.config.role_arn)
+                .role_session_name(session_name)
+                .policy(&policy)
+                .duration_seconds(duration_secs);
+            if let Some(external_id) = external_id {
+                request = request.external_id(external_id);
+            }
+            let response = request.send().await.map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "AssumeRole failed for role '{}': {}",
+                        self.config.role_arn,
+                        full_error_chain(&e)
+                    ),
+                })
+            })?;
+            response.credentials().cloned()
+        };
+
+        self.extract_credentials(credentials.as_ref(), bucket, prefix, permission)
+    }
+
     async fn vend_with_api_key(
         &self,
         bucket: &str,
@@ -452,42 +574,19 @@ impl AwsCredentialVendor {
                 })
             })?;
 
-        let policy = Self::build_policy(bucket, prefix, permission);
+        // The api_key authorizes the client and picks the permission; the AWS
+        // assume itself goes through `assume_scoped` (pod web-identity when
+        // configured, else chained AssumeRole with the key hash as external_id).
         let session_name = Self::cap_session_name(&format!("lance-api-{}", &key_hash[..16]));
-        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
-
-        debug!(
-            "AWS AssumeRole with API key: role={}, session={}, permission={}",
-            self.config.role_arn, session_name, permission
-        );
-
-        let request = self
-            .sts_client
-            .assume_role()
-            .role_arn(&self.config.role_arn)
-            .role_session_name(&session_name)
-            .policy(&policy)
-            .duration_seconds(duration_secs)
-            .external_id(&key_hash); // Use hash as external_id
-
-        let response = request.send().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!(
-                    "AssumeRole with API key failed for role '{}': {}",
-                    self.config.role_arn, e
-                ),
-            })
-        })?;
-
-        self.extract_credentials(response.credentials(), bucket, prefix, permission)
+        self.assume_scoped(bucket, prefix, permission, &session_name, Some(&key_hash))
+            .await
     }
 
-    /// Vend credentials using AssumeRole with static configuration.
+    /// Vend credentials using the vendor's static (default) permission.
     async fn vend_with_static_config(
         &self,
         bucket: &str,
         prefix: &str,
-        policy: &str,
     ) -> Result<VendedCredentials> {
         let role_session_name = self
             .config
@@ -496,40 +595,14 @@ impl AwsCredentialVendor {
             .unwrap_or_else(|| "lance-credential-vending".to_string());
         let role_session_name = Self::cap_session_name(&role_session_name);
 
-        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
-
-        debug!(
-            "AWS AssumeRole (static): role={}, session={}, permission={}",
-            self.config.role_arn, role_session_name, self.config.permission
-        );
-
-        let mut request = self
-            .sts_client
-            .assume_role()
-            .role_arn(&self.config.role_arn)
-            .role_session_name(&role_session_name)
-            .policy(policy)
-            .duration_seconds(duration_secs);
-
-        if let Some(ref external_id) = self.config.external_id {
-            request = request.external_id(external_id);
-        }
-
-        let response = request.send().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!(
-                    "AssumeRole failed for role '{}': {}",
-                    self.config.role_arn, e
-                ),
-            })
-        })?;
-
-        self.extract_credentials(
-            response.credentials(),
+        self.assume_scoped(
             bucket,
             prefix,
             self.config.permission,
+            &role_session_name,
+            self.config.external_id.as_deref(),
         )
+        .await
     }
 }
 
@@ -575,10 +648,8 @@ impl CredentialVendor for AwsCredentialVendor {
                 .into())
             }
             None => {
-                // Use AssumeRole with static configuration
-                let policy = Self::build_policy(&bucket, &prefix, self.config.permission);
-                self.vend_with_static_config(&bucket, &prefix, &policy)
-                    .await
+                // Use the vendor's static (default) permission
+                self.vend_with_static_config(&bucket, &prefix).await
             }
         }
     }
@@ -743,6 +814,41 @@ mod tests {
         assert_eq!(config.duration_millis, 7200000);
         assert_eq!(config.role_session_name, Some("my-session".to_string()));
         assert_eq!(config.region, Some("us-west-2".to_string()));
+        // Defaults to the legacy chained AssumeRole path.
+        assert_eq!(config.pod_web_identity_token_file, None);
+
+        let pod_config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole")
+            .with_pod_web_identity_token_file("/var/run/secrets/.../token");
+        assert_eq!(
+            pod_config.pod_web_identity_token_file,
+            Some("/var/run/secrets/.../token".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pod_web_identity_path_reads_token_file() {
+        // When pod_web_identity_token_file is set, the scoped assume takes the
+        // AssumeRoleWithWebIdentity branch and reads the token file first. Point
+        // it at a missing file so we deterministically hit the read error without
+        // needing a live STS -- this proves the branch selection + file read.
+        let sdk_config = aws_config::SdkConfig::builder()
+            .behavior_version(aws_config::BehaviorVersion::latest())
+            .region(aws_config::Region::new("us-east-2"))
+            .build();
+        let sts_client = StsClient::new(&sdk_config);
+        let config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole")
+            .with_pod_web_identity_token_file("/nonexistent/pod/web-identity-token");
+        let vendor = AwsCredentialVendor::with_sts_client(config, sts_client);
+
+        let err = vendor
+            .vend_credentials("s3://bucket/prefix", None)
+            .await
+            .expect_err("missing token file must fail before any STS call");
+        assert!(
+            err.to_string()
+                .contains("failed to read pod web identity token"),
+            "unexpected error: {err}"
+        );
     }
 
     // ============================================================================
diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs
index dc2d83cf278..1d2fa8cb13d 100644
--- a/rust/lance-namespace-impls/src/dir.rs
+++ b/rust/lance-namespace-impls/src/dir.rs
@@ -34,6 +34,7 @@ use lance_index::vector::{
     sq::builder::SQBuildParams,
 };
 use lance_index::{IndexType, is_system_index};
+use lance_io::object_store::throttle::is_throttle_error;
 use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
 use lance_linalg::distance::MetricType;
 use lance_table::io::commit::{ManifestNamingScheme, VERSIONS_DIR};
@@ -43,6 +44,7 @@ use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, PutM
 use std::collections::HashMap;
 use std::io::Cursor;
 use std::sync::{Arc, Mutex};
+use tokio::sync::OnceCell;
 
 use crate::context::DynamicContextProvider;
 use lance_namespace::models::{
@@ -77,6 +79,9 @@ use lance_namespace::models::{
 };
 
 use lance_core::{Error, Result, box_error};
+use lance_index::scalar::inverted::query::{
+    BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery,
+};
 use lance_namespace::LanceNamespace;
 use lance_namespace::error::NamespaceError;
 use lance_namespace::schema::arrow_schema_to_json;
@@ -738,7 +743,7 @@ impl DirectoryNamespaceBuilder {
             Self::initialize_object_store(&self.root, &self.storage_options, &self.session).await?;
 
         let manifest_ns = if self.manifest_enabled {
-            match manifest::ManifestNamespace::from_directory(
+            match manifest::ManifestNamespace::open_from_directory(
                 self.root.clone(),
                 self.storage_options.clone(),
                 self.session.clone(),
@@ -757,18 +762,19 @@ impl DirectoryNamespaceBuilder {
                     // degrading to a directory-listing view that ignores it.
                     return Err(e);
                 }
-                Err(e) => {
-                    // Failed to initialize manifest namespace, fall back to directory listing only
-                    log::warn!(
-                        "Failed to initialize manifest namespace, falling back to directory listing only: {}",
-                        e
-                    );
+                Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => {
+                    log::debug!("Manifest namespace does not exist yet: {}", e);
                     None
                 }
+                Err(e) => return Err(e),
             }
         } else {
             None
         };
+        let manifest_cell = OnceCell::new();
+        if let Some(manifest_ns) = manifest_ns {
+            let _ = manifest_cell.set(manifest_ns);
+        }
 
         // Create credential vendor once during initialization if enabled
         let credential_vendor = if has_credential_vendor_config(&self.credential_vendor_properties)
@@ -792,8 +798,12 @@ impl DirectoryNamespaceBuilder {
             session: self.session,
             object_store,
             base_path,
-            manifest_ns,
+            manifest_ns: manifest_cell,
+            write_manifest_ns: OnceCell::new(),
+            manifest_enabled: self.manifest_enabled,
             dir_listing_enabled: self.dir_listing_enabled,
+            inline_optimization_enabled: self.inline_optimization_enabled,
+            commit_retries: self.commit_retries,
             dir_listing_to_manifest_migration_enabled: self
                 .dir_listing_to_manifest_migration_enabled,
             table_version_tracking_enabled: self.table_version_tracking_enabled,
@@ -870,8 +880,12 @@ pub struct DirectoryNamespace {
     session: Option<Arc<Session>>,
     object_store: Arc<ObjectStore>,
     base_path: Path,
-    manifest_ns: Option<Arc<manifest::ManifestNamespace>>,
+    manifest_ns: OnceCell<Arc<manifest::ManifestNamespace>>,
+    write_manifest_ns: OnceCell<Arc<manifest::ManifestNamespace>>,
+    manifest_enabled: bool,
     dir_listing_enabled: bool,
+    inline_optimization_enabled: bool,
+    commit_retries: Option<u32>,
     /// When true, root-level table operations check the manifest first before
     /// falling back to directory listing. When false, root-level tables skip
     /// the manifest check and use directory listing directly.
@@ -983,6 +997,52 @@ impl TransactionAlteration {
 }
 
 impl DirectoryNamespace {
+    fn manifest_ns_for_read(&self) -> Option<&Arc<manifest::ManifestNamespace>> {
+        self.write_manifest_ns
+            .get()
+            .or_else(|| self.manifest_ns.get())
+    }
+
+    async fn manifest_ns_for_write(&self) -> Result<Option<Arc<manifest::ManifestNamespace>>> {
+        if !self.manifest_enabled {
+            return Ok(None);
+        }
+
+        let manifest_ns = self
+            .write_manifest_ns
+            .get_or_try_init(|| async {
+                manifest::ManifestNamespace::from_directory(
+                    self.root.clone(),
+                    self.storage_options.clone(),
+                    self.session.clone(),
+                    self.object_store.clone(),
+                    self.base_path.clone(),
+                    self.dir_listing_enabled,
+                    self.inline_optimization_enabled,
+                    self.commit_retries,
+                )
+                .await
+                .map(Arc::new)
+            })
+            .await?;
+        Ok(Some(manifest_ns.clone()))
+    }
+
+    fn child_namespace_requires_manifest_error(&self) -> Error {
+        if self.manifest_enabled {
+            NamespaceError::NamespaceNotFound {
+                message: "Child namespace reads require an existing __manifest dataset".to_string(),
+            }
+            .into()
+        } else {
+            NamespaceError::Unsupported {
+                message: "Child namespaces are only supported when manifest mode is enabled"
+                    .to_string(),
+            }
+            .into()
+        }
+    }
+
     /// Apply pagination to a list of table names
     ///
     /// Sorts the list alphabetically and applies pagination using page_token (start_after) and limit.
@@ -1056,7 +1116,7 @@ impl DirectoryNamespace {
             let table_name = &path[..path.len() - 6];
 
             // Use atomic check to skip deregistered tables.
-            let status = self.check_table_status(table_name).await;
+            let status = self.check_table_status(table_name).await?;
             if status.is_deregistered {
                 continue;
             }
@@ -1407,17 +1467,19 @@ impl DirectoryNamespace {
             .load()
             .await
             .map_err(|e| {
-                lance_core::Error::from(NamespaceError::TableNotFound {
-                    message: format!(
-                        "branch '{}' not found for table at '{}': {}",
-                        branch, table_uri, e
-                    ),
-                })
+                let message = format!(
+                    "branch '{}' not found for table at '{}': {}",
+                    branch, table_uri, e
+                );
+                Self::map_open_error(e, NamespaceError::TableNotFound { message })
             })?;
-        dataset.branches().get(branch).await.map_err(|_| {
-            lance_core::Error::from(NamespaceError::TableNotFound {
-                message: format!("branch '{}' not found for table at '{}'", branch, table_uri),
-            })
+        dataset.branches().get(branch).await.map_err(|e| {
+            Self::map_open_error(
+                e,
+                NamespaceError::TableNotFound {
+                    message: format!("branch '{}' not found for table at '{}'", branch, table_uri),
+                },
+            )
         })?;
         Ok(dataset)
     }
@@ -1446,9 +1508,8 @@ impl DirectoryNamespace {
             .load()
             .await
             .map_err(|e| {
-                lance_core::Error::from(NamespaceError::TableNotFound {
-                    message: format!("table at '{}' not found: {}", table_uri, e),
-                })
+                let message = format!("table at '{}' not found: {}", table_uri, e);
+                Self::map_open_error(e, NamespaceError::TableNotFound { message })
             })?;
         let branch_location = main.branch_location().find_branch(Some(branch))?;
         match main.branches().get(branch).await {
@@ -1613,9 +1674,9 @@ impl DirectoryNamespace {
 
         if needs_sort {
             if descending {
-                table_versions.sort_by(|a, b| b.version.cmp(&a.version));
+                table_versions.sort_by_key(|v| std::cmp::Reverse(v.version));
             } else {
-                table_versions.sort_by(|a, b| a.version.cmp(&b.version));
+                table_versions.sort_by_key(|v| v.version);
             }
         }
 
@@ -1634,10 +1695,11 @@ impl DirectoryNamespace {
         request: DescribeTableRequest,
     ) -> Result<DescribeTableResponse> {
         let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1);
         let skip_manifest_for_root = self.dir_listing_enabled
             && is_root_level
             && !self.dir_listing_to_manifest_migration_enabled;
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !skip_manifest_for_root
         {
             match manifest_ns.describe_table(request.clone()).await {
@@ -1661,19 +1723,31 @@ impl DirectoryNamespace {
                     // rather than degrading to a directory-listing view.
                     return Err(e);
                 }
-                Err(_) if self.dir_listing_enabled && is_root_level => {
-                    // Fall through to directory check only for single-level IDs
+                Err(e) if self.dir_listing_enabled && is_root_level => {
+                    // Only a genuinely-absent table (e.g. an unmigrated on-disk
+                    // table) may fall through to the directory check; any other
+                    // manifest error must propagate rather than be read as missing.
+                    if !Self::is_manifest_table_absent_error(&e) {
+                        return Err(Self::classify_storage_error(e));
+                    }
                 }
                 Err(e) => return Err(e),
             }
         }
+        if is_child_table {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
 
         let table_name = Self::table_name_from_id(&request.id)?;
         let table_id = Self::format_table_id_from_request(&request.id);
+        if !self.dir_listing_enabled {
+            return Err(NamespaceError::TableNotFound { message: table_id }.into());
+        }
+
         let table_uri = self.table_full_uri(&table_name);
 
         // Atomically check table existence and deregistration status
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
 
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
@@ -1774,12 +1848,14 @@ impl DirectoryNamespace {
                         .checkout_version(requested_version as u64)
                         .await
                         .map_err(|e| {
-                            lance_core::Error::from(NamespaceError::TableVersionNotFound {
-                                message: format!(
-                                    "Version {} not found for table '{}': {}",
-                                    requested_version, table_name, e
-                                ),
-                            })
+                            let message = format!(
+                                "Version {} not found for table '{}': {}",
+                                requested_version, table_name, e
+                            );
+                            Self::map_open_error(
+                                e,
+                                NamespaceError::TableVersionNotFound { message },
+                            )
                         })?;
                 }
 
@@ -1893,22 +1969,20 @@ impl DirectoryNamespace {
         let builder = self.configured_builder(table_uri);
 
         let dataset = builder.load().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::TableNotFound {
-                message: format!(
-                    "Failed to open table at '{}' for {}: {}",
-                    table_uri, operation, e
-                ),
-            })
+            let message = format!(
+                "Failed to open table at '{}' for {}: {}",
+                table_uri, operation, e
+            );
+            Self::map_open_error(e, NamespaceError::TableNotFound { message })
         })?;
 
         if let Some(version) = version {
             return dataset.checkout_version(version as u64).await.map_err(|e| {
-                lance_core::Error::from(NamespaceError::TableVersionNotFound {
-                    message: format!(
-                        "Failed to checkout version {} for table at '{}' during {}: {}",
-                        version, table_uri, operation, e
-                    ),
-                })
+                let message = format!(
+                    "Failed to checkout version {} for table at '{}' during {}: {}",
+                    version, table_uri, operation, e
+                );
+                Self::map_open_error(e, NamespaceError::TableVersionNotFound { message })
             });
         }
 
@@ -2339,7 +2413,7 @@ impl DirectoryNamespace {
     }
 
     fn table_full_uri(&self, table_name: &str) -> String {
-        format!("{}/{}.lance", &self.root, table_name)
+        format!("{}/{}.lance", self.root, table_name)
     }
 
     /// Get the object store path for a table (relative to base_path)
@@ -2370,27 +2444,107 @@ impl DirectoryNamespace {
     /// This performs a single directory listing to get a consistent snapshot of the
     /// table's state, avoiding race conditions between checking existence and
     /// checking deregistration status.
-    pub(crate) async fn check_table_status(&self, table_name: &str) -> TableStatus {
+    pub(crate) async fn check_table_status(&self, table_name: &str) -> Result<TableStatus> {
         let table_path = self.table_path(table_name);
         match self.object_store.read_dir(table_path).await {
             Ok(entries) => {
                 let exists = !entries.is_empty();
                 let is_deregistered = entries.iter().any(|e| e.ends_with(".lance-deregistered"));
                 let has_reserved_file = entries.iter().any(|e| e.ends_with(".lance-reserved"));
-                TableStatus {
+                Ok(TableStatus {
                     exists,
                     is_deregistered,
                     has_reserved_file,
-                }
+                })
             }
-            Err(_) => TableStatus {
+            // Local filesystems error on a missing directory where object stores
+            // return an empty listing; both mean the table does not exist.
+            Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => Ok(TableStatus {
                 exists: false,
                 is_deregistered: false,
                 has_reserved_file: false,
-            },
+            }),
+            // Any other failure must propagate: collapsing it to "does not exist"
+            // lets a transient error overwrite a live table via create/exist-ok
+            // callers and destroys the retry evidence classifiers depend on.
+            Err(e) => Err(Self::classify_storage_error(e)),
         }
     }
 
+    /// Classify a storage error into a typed [`NamespaceError`]. The full source
+    /// text is embedded in the message because the pyo3 layer flattens namespace
+    /// errors to message-only (no `__cause__`), so that is the only place the
+    /// 429/503 evidence survives to Python.
+    fn classify_storage_error(err: Error) -> Error {
+        if matches!(&err, Error::Namespace { .. }) {
+            return err;
+        }
+        let detail = err.to_string();
+        if let Error::IO { source, .. } = &err
+            && let Some(os_err) = source.downcast_ref::<ObjectStoreError>()
+        {
+            if is_throttle_error(os_err) {
+                return NamespaceError::Throttling {
+                    message: format!(
+                        "Storage request was throttled while resolving table: {detail}"
+                    ),
+                }
+                .into();
+            }
+            if Self::is_service_unavailable_error(os_err) {
+                return NamespaceError::ServiceUnavailable {
+                    message: format!("Storage service unavailable while resolving table: {detail}"),
+                }
+                .into();
+            }
+        }
+        NamespaceError::Internal {
+            message: format!("Storage error while resolving table: {detail}"),
+        }
+        .into()
+    }
+
+    /// Detect a clearly-transient 5xx not already caught by [`is_throttle_error`].
+    /// `object_store` does not expose HTTP status codes, so match the (deliberately
+    /// narrow) canonical status phrases in the message.
+    fn is_service_unavailable_error(err: &ObjectStoreError) -> bool {
+        if let ObjectStoreError::Generic { source, .. } = err {
+            let message = source.to_string().to_ascii_lowercase();
+            message.contains("503 service unavailable")
+                || message.contains("502 bad gateway")
+                || message.contains("504 gateway timeout")
+        } else {
+            false
+        }
+    }
+
+    /// Whether a manifest error means the table is genuinely absent (rather than a
+    /// storage failure while consulting the manifest). Only such errors may fall
+    /// through to the directory listing; anything else must propagate.
+    fn is_manifest_table_absent_error(err: &Error) -> bool {
+        if manifest::ManifestNamespace::is_not_found_load_error(err) {
+            return true;
+        }
+        if let Error::Namespace { source, .. } = err
+            && let Some(ns_err) = source.downcast_ref::<NamespaceError>()
+        {
+            return matches!(ns_err, NamespaceError::TableNotFound { .. });
+        }
+        false
+    }
+
+    /// Map a dataset/version/branch open error: a transient IO error propagates
+    /// typed via [`classify_storage_error`], while a genuine not-found (missing
+    /// dataset, version, or ref) keeps the caller's `not_found` variant.
+    fn map_open_error(err: Error, not_found: NamespaceError) -> Error {
+        if matches!(&err, Error::IO { .. })
+            && !manifest::ManifestNamespace::is_not_found_load_error(&err)
+        {
+            return Self::classify_storage_error(err);
+        }
+        not_found.into()
+    }
+
     async fn put_marker_file_atomic(
         &self,
         path: &Path,
@@ -2525,7 +2679,7 @@ impl DirectoryNamespace {
     /// - Manifest registration fails
     pub async fn migrate(&self) -> Result<usize> {
         // We only care about tables in the root namespace
-        let Some(ref manifest_ns) = self.manifest_ns else {
+        let Some(manifest_ns) = self.manifest_ns_for_write().await? else {
             return Ok(0); // No manifest, nothing to migrate
         };
 
@@ -2797,10 +2951,13 @@ impl LanceNamespace for DirectoryNamespace {
         request: ListNamespacesRequest,
     ) -> Result<ListNamespacesResponse> {
         self.record_op("list_namespaces");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.list_namespaces(request).await;
         }
 
+        if request.id.as_ref().is_some_and(|id| !id.is_empty()) {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
         Self::validate_root_namespace_id(&request.id)?;
         Ok(ListNamespacesResponse::new(vec![]))
     }
@@ -2810,10 +2967,13 @@ impl LanceNamespace for DirectoryNamespace {
         request: DescribeNamespaceRequest,
     ) -> Result<DescribeNamespaceResponse> {
         self.record_op("describe_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.describe_namespace(request).await;
         }
 
+        if request.id.as_ref().is_some_and(|id| !id.is_empty()) {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
         Self::validate_root_namespace_id(&request.id)?;
         #[allow(clippy::needless_update)]
         Ok(DescribeNamespaceResponse {
@@ -2827,7 +2987,7 @@ impl LanceNamespace for DirectoryNamespace {
         request: CreateNamespaceRequest,
     ) -> Result<CreateNamespaceResponse> {
         self.record_op("create_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.create_namespace(request).await;
         }
 
@@ -2847,7 +3007,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
         self.record_op("drop_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.drop_namespace(request).await;
         }
 
@@ -2867,7 +3027,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
         self.record_op("namespace_exists");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.namespace_exists(request).await;
         }
 
@@ -2875,11 +3035,7 @@ impl LanceNamespace for DirectoryNamespace {
             return Ok(());
         }
 
-        Err(NamespaceError::NamespaceNotFound {
-            message: "Child namespaces are only supported when manifest mode is enabled"
-                .to_string(),
-        }
-        .into())
+        Err(self.child_namespace_requires_manifest_error())
     }
 
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
@@ -2893,31 +3049,30 @@ impl LanceNamespace for DirectoryNamespace {
 
         // For child namespaces, always delegate to manifest (if enabled)
         if !namespace_id.is_empty() {
-            if let Some(ref manifest_ns) = self.manifest_ns {
+            if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 return manifest_ns.list_tables(request).await;
             }
-            return Err(NamespaceError::Unsupported {
-                message: "Child namespaces are only supported when manifest mode is enabled"
-                    .to_string(),
-            }
-            .into());
+            return Err(self.child_namespace_requires_manifest_error());
         }
 
         // When only manifest is enabled (no directory listing), delegate directly to manifest
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !self.dir_listing_enabled
         {
             return manifest_ns.list_tables(request).await;
         }
+        if !self.dir_listing_enabled {
+            return Ok(ListTablesResponse::new(vec![]));
+        }
 
         // When both manifest and directory listing are enabled with migration mode,
         // we need to merge and deduplicate
-        let mut tables = if self.manifest_ns.is_some()
+        let mut tables = if self.manifest_ns_for_read().is_some()
             && self.dir_listing_enabled
             && self.dir_listing_to_manifest_migration_enabled
         {
             // Get all manifest table locations (for deduplication)
-            let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns {
+            let manifest_locations = if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 manifest_ns.list_manifest_table_locations().await?
             } else {
                 std::collections::HashSet::new()
@@ -2927,7 +3082,7 @@ impl LanceNamespace for DirectoryNamespace {
             let mut manifest_request = request.clone();
             manifest_request.limit = None;
             manifest_request.page_token = None;
-            let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns {
+            let manifest_tables = if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 let manifest_response = manifest_ns.list_tables(manifest_request).await?;
                 manifest_response.tables
             } else {
@@ -2975,10 +3130,11 @@ impl LanceNamespace for DirectoryNamespace {
     async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
         self.record_op("table_exists");
         let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1);
         let skip_manifest_for_root = self.dir_listing_enabled
             && is_root_level
             && !self.dir_listing_to_manifest_migration_enabled;
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !skip_manifest_for_root
         {
             match manifest_ns.table_exists(request.clone()).await {
@@ -2988,18 +3144,29 @@ impl LanceNamespace for DirectoryNamespace {
                     // rather than degrading to a directory-listing view.
                     return Err(e);
                 }
-                Err(_) if self.dir_listing_enabled && is_root_level => {
-                    // Fall through to directory check only for single-level IDs
+                Err(e) if self.dir_listing_enabled && is_root_level => {
+                    // Only a genuinely-absent table (e.g. an unmigrated on-disk
+                    // table) may fall through to the directory check; any other
+                    // manifest error must propagate rather than be read as missing.
+                    if !Self::is_manifest_table_absent_error(&e) {
+                        return Err(Self::classify_storage_error(e));
+                    }
                 }
                 Err(e) => return Err(e),
             }
         }
+        if is_child_table {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
 
         let table_name = Self::table_name_from_id(&request.id)?;
         let table_id = Self::format_table_id_from_request(&request.id);
+        if !self.dir_listing_enabled {
+            return Err(NamespaceError::TableNotFound { message: table_id }.into());
+        }
 
         // Atomically check table existence and deregistration status
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
 
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
@@ -3020,7 +3187,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
         self.record_op("drop_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.drop_table(request).await;
         }
 
@@ -3050,7 +3217,7 @@ impl LanceNamespace for DirectoryNamespace {
         request_data: Bytes,
     ) -> Result<CreateTableResponse> {
         self.record_op("create_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.create_table(request, request_data).await;
         }
 
@@ -3058,7 +3225,7 @@ impl LanceNamespace for DirectoryNamespace {
 
         let table_name = Self::table_name_from_id(&request.id)?;
         let table_uri = self.table_full_uri(&table_name);
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
         let (reader, _num_rows) =
             Self::ipc_reader_from_request_data(&request_data, "create_table")?;
 
@@ -3097,7 +3264,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
         self.record_op("declare_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             let mut response = manifest_ns.declare_table(request.clone()).await?;
             if let Some(ref location) = response.location {
                 // For backwards compatibility, only skip vending credentials when explicitly set to false
@@ -3136,7 +3303,7 @@ impl LanceNamespace for DirectoryNamespace {
         // Check if table already has data (created via create_table).
         // The atomic put only prevents races between concurrent declare_table calls,
         // not between declare_table and existing data.
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
         if status.exists && !status.has_reserved_file {
             // Table has data but no reserved file - it was created with data
             return Err(NamespaceError::TableAlreadyExists {
@@ -3188,7 +3355,7 @@ impl LanceNamespace for DirectoryNamespace {
     ) -> Result<lance_namespace::models::RegisterTableResponse> {
         self.record_op("register_table");
         // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return LanceNamespace::register_table(manifest_ns.as_ref(), request).await;
         }
 
@@ -3205,7 +3372,7 @@ impl LanceNamespace for DirectoryNamespace {
     ) -> Result<lance_namespace::models::DeregisterTableResponse> {
         self.record_op("deregister_table");
         // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await;
         }
 
@@ -3215,7 +3382,7 @@ impl LanceNamespace for DirectoryNamespace {
 
         // Check table existence and deregistration status.
         // This provides better error messages for common cases.
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
 
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
@@ -3263,7 +3430,7 @@ impl LanceNamespace for DirectoryNamespace {
         &self,
         request: AlterTableAddColumnsRequest,
     ) -> Result<AlterTableAddColumnsResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.alter_table_add_columns(request).await;
         }
 
@@ -3272,7 +3439,7 @@ impl LanceNamespace for DirectoryNamespace {
         let table_uri = self.table_full_uri(&table_name);
 
         // Check table existence and deregistration status before opening the dataset
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
                 message: table_name,
@@ -3321,7 +3488,7 @@ impl LanceNamespace for DirectoryNamespace {
         &self,
         request: AlterTableAlterColumnsRequest,
     ) -> Result<AlterTableAlterColumnsResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.alter_table_alter_columns(request).await;
         }
 
@@ -3329,7 +3496,7 @@ impl LanceNamespace for DirectoryNamespace {
         let table_uri = self.table_full_uri(&table_name);
 
         // Check table existence and deregistration status before opening the dataset
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
                 message: table_name,
@@ -3371,7 +3538,7 @@ impl LanceNamespace for DirectoryNamespace {
         &self,
         request: AlterTableDropColumnsRequest,
     ) -> Result<AlterTableDropColumnsResponse> {
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.alter_table_drop_columns(request).await;
         }
 
@@ -3379,7 +3546,7 @@ impl LanceNamespace for DirectoryNamespace {
         let table_uri = self.table_full_uri(&table_name);
 
         // Check table existence and deregistration status before opening the dataset
-        let status = self.check_table_status(&table_name).await;
+        let status = self.check_table_status(&table_name).await?;
         if !status.exists {
             return Err(NamespaceError::TableNotFound {
                 message: table_name,
@@ -4896,14 +5063,21 @@ impl LanceNamespace for DirectoryNamespace {
                         })?;
                 }
 
+                scanner
+                    .full_text_search(fts)
+                    .map_err(|e| NamespaceError::InvalidInput {
+                        message: format!("Invalid full text search: {:?}", e),
+                    })?;
+            } else if let Some(ref structured_query) = fts_query.structured_query {
+                // Structured FTS: map the namespace query model into the engine FtsQuery.
+                let engine_query = build_engine_fts_query(&structured_query.query)?;
+                let fts = FullTextSearchQuery::new_query(engine_query);
                 scanner
                     .full_text_search(fts)
                     .map_err(|e| NamespaceError::InvalidInput {
                         message: format!("Invalid full text search: {:?}", e),
                     })?;
             }
-            // Note: structured_query would require more complex parsing
-            // For now, we only support string_query
         }
 
         // Apply column projection if specified
@@ -5319,10 +5493,227 @@ impl LanceNamespace for DirectoryNamespace {
     }
 }
 
+/// Maps a namespace structured `FtsQuery` model into the engine `FtsQuery`. Mirrors the mapping the
+/// JNI scanner performs, so the local `queryTable` path honors `structured_query` the same way a
+/// `fragment.newScan(fullTextQuery)` does.
+fn build_engine_fts_query(
+    query: &lance_namespace::models::FtsQuery,
+) -> std::result::Result<FtsQuery, NamespaceError> {
+    if let Some(ref m) = query.r#match {
+        Ok(FtsQuery::Match(build_engine_match_query(m)?))
+    } else if let Some(ref p) = query.phrase {
+        let mut phrase = PhraseQuery::new(p.terms.clone());
+        if let Some(ref column) = p.column {
+            phrase = phrase.with_column(Some(column.clone()));
+        }
+        if let Some(slop) = p.slop {
+            phrase = phrase.with_slop(slop as u32);
+        }
+        Ok(FtsQuery::Phrase(phrase))
+    } else if let Some(ref mm) = query.multi_match {
+        let match_queries = mm
+            .match_queries
+            .iter()
+            .map(build_engine_match_query)
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+        Ok(FtsQuery::MultiMatch(MultiMatchQuery { match_queries }))
+    } else if let Some(ref b) = query.boolean {
+        let mut clauses: Vec<(Occur, FtsQuery)> = Vec::new();
+        for clause in &b.must {
+            clauses.push((Occur::Must, build_engine_fts_query(clause)?));
+        }
+        for clause in &b.should {
+            clauses.push((Occur::Should, build_engine_fts_query(clause)?));
+        }
+        for clause in &b.must_not {
+            clauses.push((Occur::MustNot, build_engine_fts_query(clause)?));
+        }
+        Ok(FtsQuery::Boolean(BooleanQuery::new(clauses)))
+    } else if let Some(ref boost) = query.boost {
+        let positive = build_engine_fts_query(&boost.positive)?;
+        let negative = build_engine_fts_query(&boost.negative)?;
+        Ok(FtsQuery::Boost(BoostQuery::new(
+            positive,
+            negative,
+            boost.negative_boost,
+        )))
+    } else {
+        Err(NamespaceError::InvalidInput {
+            message: "structured_query.query must set exactly one of match, phrase, multi_match, \
+                      boolean, or boost"
+                .to_string(),
+        })
+    }
+}
+
+fn build_engine_match_query(
+    m: &lance_namespace::models::MatchQuery,
+) -> std::result::Result<MatchQuery, NamespaceError> {
+    let mut match_query = MatchQuery::new(m.terms.clone());
+    if let Some(ref column) = m.column {
+        match_query = match_query.with_column(Some(column.clone()));
+    }
+    if let Some(boost) = m.boost {
+        match_query = match_query.with_boost(boost);
+    }
+    if let Some(fuzziness) = m.fuzziness {
+        match_query = match_query.with_fuzziness(Some(fuzziness as u32));
+    }
+    if let Some(max_expansions) = m.max_expansions {
+        match_query = match_query.with_max_expansions(max_expansions as usize);
+    }
+    if let Some(ref operator) = m.operator {
+        let op =
+            Operator::try_from(operator.as_str()).map_err(|e| NamespaceError::InvalidInput {
+                message: format!("Invalid FTS operator: {:?}", e),
+            })?;
+        match_query = match_query.with_operator(op);
+    }
+    if let Some(prefix_length) = m.prefix_length {
+        match_query = match_query.with_prefix_length(prefix_length as u32);
+    }
+    Ok(match_query)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use arrow_ipc::reader::{FileReader, StreamReader};
+
+    #[test]
+    fn test_build_engine_fts_query_match() {
+        let mut ns_match = lance_namespace::models::MatchQuery::new("hello world".to_string());
+        ns_match.column = Some("body".to_string());
+        ns_match.operator = Some("AND".to_string());
+        ns_match.fuzziness = Some(1);
+        ns_match.max_expansions = Some(30);
+        ns_match.boost = Some(2.0);
+        ns_match.prefix_length = Some(2);
+
+        let mut ns_query = lance_namespace::models::FtsQuery::new();
+        ns_query.r#match = Some(Box::new(ns_match));
+
+        match build_engine_fts_query(&ns_query).unwrap() {
+            FtsQuery::Match(m) => {
+                assert_eq!(m.terms, "hello world");
+                assert_eq!(m.column, Some("body".to_string()));
+                assert_eq!(m.operator, Operator::And);
+                assert_eq!(m.fuzziness, Some(1));
+                assert_eq!(m.max_expansions, 30);
+                assert_eq!(m.boost, 2.0);
+                assert_eq!(m.prefix_length, 2);
+            }
+            other => panic!("expected Match, got {:?}", other),
+        }
+    }
+
+    /// Wraps a namespace `MatchQuery` (with a column) as an `FtsQuery` for use as a clause in
+    /// compound queries (boolean / boost).
+    fn ns_match_query(terms: &str, column: &str) -> lance_namespace::models::FtsQuery {
+        let mut m = lance_namespace::models::MatchQuery::new(terms.to_string());
+        m.column = Some(column.to_string());
+        let mut q = lance_namespace::models::FtsQuery::new();
+        q.r#match = Some(Box::new(m));
+        q
+    }
+
+    #[test]
+    fn test_build_engine_fts_query_phrase() {
+        let mut ns_phrase = lance_namespace::models::PhraseQuery::new("hello world".to_string());
+        ns_phrase.column = Some("body".to_string());
+        ns_phrase.slop = Some(2);
+
+        let mut ns_query = lance_namespace::models::FtsQuery::new();
+        ns_query.phrase = Some(Box::new(ns_phrase));
+
+        match build_engine_fts_query(&ns_query).unwrap() {
+            FtsQuery::Phrase(p) => {
+                assert_eq!(p.terms, "hello world");
+                assert_eq!(p.column, Some("body".to_string()));
+                assert_eq!(p.slop, 2);
+            }
+            other => panic!("expected Phrase, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_build_engine_fts_query_multi_match() {
+        let mut m1 = lance_namespace::models::MatchQuery::new("hello".to_string());
+        m1.column = Some("title".to_string());
+        let mut m2 = lance_namespace::models::MatchQuery::new("hello".to_string());
+        m2.column = Some("body".to_string());
+        m2.boost = Some(2.0);
+
+        let ns_multi = lance_namespace::models::MultiMatchQuery::new(vec![m1, m2]);
+        let mut ns_query = lance_namespace::models::FtsQuery::new();
+        ns_query.multi_match = Some(Box::new(ns_multi));
+
+        match build_engine_fts_query(&ns_query).unwrap() {
+            FtsQuery::MultiMatch(mm) => {
+                assert_eq!(mm.match_queries.len(), 2);
+                assert_eq!(mm.match_queries[0].terms, "hello");
+                assert_eq!(mm.match_queries[0].column, Some("title".to_string()));
+                assert_eq!(mm.match_queries[1].column, Some("body".to_string()));
+                assert_eq!(mm.match_queries[1].boost, 2.0);
+            }
+            other => panic!("expected MultiMatch, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_build_engine_fts_query_boolean() {
+        // BooleanQuery::new(must, must_not, should)
+        let ns_boolean = lance_namespace::models::BooleanQuery::new(
+            vec![ns_match_query("must-term", "body")],
+            vec![ns_match_query("must-not-term", "body")],
+            vec![ns_match_query("should-term", "body")],
+        );
+        let mut ns_query = lance_namespace::models::FtsQuery::new();
+        ns_query.boolean = Some(Box::new(ns_boolean));
+
+        match build_engine_fts_query(&ns_query).unwrap() {
+            FtsQuery::Boolean(b) => {
+                assert!(matches!(&b.must[..], [FtsQuery::Match(m)] if m.terms == "must-term"));
+                assert!(
+                    matches!(&b.must_not[..], [FtsQuery::Match(m)] if m.terms == "must-not-term")
+                );
+                assert!(matches!(&b.should[..], [FtsQuery::Match(m)] if m.terms == "should-term"));
+            }
+            other => panic!("expected Boolean, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_build_engine_fts_query_boost() {
+        let mut ns_boost = lance_namespace::models::BoostQuery::new(
+            ns_match_query("positive-term", "body"),
+            ns_match_query("negative-term", "body"),
+        );
+        ns_boost.negative_boost = Some(0.25);
+
+        let mut ns_query = lance_namespace::models::FtsQuery::new();
+        ns_query.boost = Some(Box::new(ns_boost));
+
+        match build_engine_fts_query(&ns_query).unwrap() {
+            FtsQuery::Boost(b) => {
+                assert!(
+                    matches!(b.positive.as_ref(), FtsQuery::Match(m) if m.terms == "positive-term")
+                );
+                assert!(
+                    matches!(b.negative.as_ref(), FtsQuery::Match(m) if m.terms == "negative-term")
+                );
+                assert_eq!(b.negative_boost, 0.25);
+            }
+            other => panic!("expected Boost, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_build_engine_fts_query_requires_a_variant() {
+        // An FtsQuery with no variant set is rejected rather than silently ignored.
+        let empty = lance_namespace::models::FtsQuery::new();
+        assert!(build_engine_fts_query(&empty).is_err());
+    }
     use lance::dataset::Dataset;
     use lance::index::DatasetIndexExt;
     use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
@@ -5465,6 +5856,514 @@ mod tests {
         Arc::new(Session::new(0, 0, registry))
     }
 
+    // Fault-injection store: returns a runtime-toggleable result from
+    // `list_with_delimiter` (the call `check_table_status` makes) and delegates
+    // everything else, so a table can be created before failures are injected.
+    use futures::stream::BoxStream;
+    use object_store::{
+        CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+        PutMultipartOptions, PutPayload, PutResult, Result as OSResult,
+    };
+    use std::ops::Range;
+
+    #[derive(Debug, Clone, Copy)]
+    enum ListBehavior {
+        Throttle,
+        ServiceUnavailable,
+        Internal,
+        NotFound,
+        EmptyListing,
+    }
+
+    #[derive(Debug)]
+    struct FailingListStore {
+        target: Arc<dyn OSObjectStore>,
+        behavior: Arc<Mutex<Option<ListBehavior>>>,
+    }
+
+    impl std::fmt::Display for FailingListStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "FailingListStore({})", self.target)
+        }
+    }
+
+    #[async_trait]
+    impl OSObjectStore for FailingListStore {
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.target.put_opts(location, bytes, opts).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.target.put_multipart_opts(location, opts).await
+        }
+
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            self.target.get_opts(location, options).await
+        }
+
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            self.target.get_ranges(location, ranges).await
+        }
+
+        fn delete_stream(
+            &self,
+            locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            self.target.delete_stream(locations)
+        }
+
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.target.list(prefix)
+        }
+
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            let behavior = *self.behavior.lock().unwrap();
+            match behavior {
+                None => self.target.list_with_delimiter(prefix).await,
+                Some(ListBehavior::EmptyListing) => Ok(ListResult {
+                    common_prefixes: Vec::new(),
+                    objects: Vec::new(),
+                }),
+                // Mirrors the object_store retry-exhaustion message shape for an
+                // Azure ServerBusy response, which is what the incident produced.
+                Some(ListBehavior::Throttle) => Err(ObjectStoreError::Generic {
+                    store: "test",
+                    source: "Error performing list request: response error, after 3 retries, \
+                             max_retries: 3, retry_timeout: 180s - HTTP status server error \
+                             (503 Service Unavailable): ServerBusy: The server is busy"
+                        .into(),
+                }),
+                Some(ListBehavior::ServiceUnavailable) => Err(ObjectStoreError::Generic {
+                    store: "test",
+                    source: "Error performing list request: 503 Service Unavailable".into(),
+                }),
+                Some(ListBehavior::Internal) => Err(ObjectStoreError::Generic {
+                    store: "test",
+                    source: "Error performing list request: catastrophic unclassified failure"
+                        .into(),
+                }),
+                Some(ListBehavior::NotFound) => Err(ObjectStoreError::NotFound {
+                    path: "test_table.lance".to_string(),
+                    source: "entity not found".into(),
+                }),
+            }
+        }
+
+        async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+            self.target.copy_opts(from, to, opts).await
+        }
+    }
+
+    #[derive(Debug)]
+    struct FailingListStoreProvider {
+        behavior: Arc<Mutex<Option<ListBehavior>>>,
+    }
+
+    #[async_trait]
+    impl lance_io::object_store::ObjectStoreProvider for FailingListStoreProvider {
+        async fn new_store(
+            &self,
+            base_path: Url,
+            params: &ObjectStoreParams,
+        ) -> Result<ObjectStore> {
+            let mut store = FileStoreProvider.new_store(base_path, params).await?;
+            store.inner = Arc::new(FailingListStore {
+                target: store.inner.clone(),
+                behavior: self.behavior.clone(),
+            });
+            Ok(store)
+        }
+
+        fn extract_path(&self, url: &Url) -> Result<Path> {
+            FileStoreProvider.extract_path(url)
+        }
+
+        fn calculate_object_store_prefix(
+            &self,
+            url: &Url,
+            storage_options: Option<&HashMap<String, String>>,
+        ) -> Result<String> {
+            FileStoreProvider.calculate_object_store_prefix(url, storage_options)
+        }
+    }
+
+    fn build_failing_list_session(behavior: Arc<Mutex<Option<ListBehavior>>>) -> Arc<Session> {
+        let registry = Arc::new(ObjectStoreRegistry::default());
+        registry.insert(
+            "file-object-store",
+            Arc::new(FailingListStoreProvider { behavior }),
+        );
+        Arc::new(Session::new(0, 0, registry))
+    }
+
+    /// Build a dir-listing namespace whose object store's listing calls follow a
+    /// shared, runtime-toggleable behavior. Returns the namespace, the temp dir
+    /// (kept alive for the store), and the behavior toggle.
+    async fn failing_list_namespace() -> (
+        DirectoryNamespace,
+        TempStdDir,
+        Arc<Mutex<Option<ListBehavior>>>,
+    ) {
+        let temp_dir = TempStdDir::default();
+        let root_uri = file_object_store_uri(temp_dir.to_str().unwrap());
+        let behavior = Arc::new(Mutex::new(None));
+        let session = build_failing_list_session(behavior.clone());
+        let namespace = DirectoryNamespaceBuilder::new(root_uri)
+            .session(session)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        (namespace, temp_dir, behavior)
+    }
+
+    async fn create_named_dir_table(namespace: &DirectoryNamespace, name: &str) {
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_req = CreateTableRequest::new();
+        create_req.id = Some(vec![name.to_string()]);
+        namespace
+            .create_table(create_req, Bytes::from(ipc_data))
+            .await
+            .unwrap();
+    }
+
+    /// Regression test for the throttling-induced TableNotFound bug: a storage
+    /// error while resolving a table must surface as a typed storage error
+    /// (Throttling / ServiceUnavailable / Internal) carrying the underlying
+    /// evidence in its message — never as TableNotFound.
+    #[tokio::test]
+    async fn test_table_resolution_propagates_storage_errors_not_table_not_found() {
+        for (behavior, expected_code, evidence) in [
+            (ListBehavior::Throttle, ErrorCode::Throttling, "serverbusy"),
+            (
+                ListBehavior::ServiceUnavailable,
+                ErrorCode::ServiceUnavailable,
+                "503 service unavailable",
+            ),
+            (ListBehavior::Internal, ErrorCode::Internal, "catastrophic"),
+        ] {
+            let (namespace, _temp_dir, toggle) = failing_list_namespace().await;
+            create_named_dir_table(&namespace, "checkpoint").await;
+            *toggle.lock().unwrap() = Some(behavior);
+
+            let mut describe_req = DescribeTableRequest::new();
+            describe_req.id = Some(vec!["checkpoint".to_string()]);
+            let err = namespace.describe_table(describe_req).await.unwrap_err();
+            let msg = err.to_string();
+            assert_eq!(
+                mutation_error_code(err),
+                expected_code,
+                "describe_table under {behavior:?}; msg: {msg}"
+            );
+            assert!(
+                msg.to_ascii_lowercase().contains(evidence),
+                "describe_table message must carry storage evidence '{evidence}', got: {msg}"
+            );
+
+            let mut exists_req = TableExistsRequest::new();
+            exists_req.id = Some(vec!["checkpoint".to_string()]);
+            let err = namespace.table_exists(exists_req).await.unwrap_err();
+            let msg = err.to_string();
+            assert_eq!(
+                mutation_error_code(err),
+                expected_code,
+                "table_exists under {behavior:?}; msg: {msg}"
+            );
+            assert!(
+                msg.to_ascii_lowercase().contains(evidence),
+                "table_exists message must carry storage evidence '{evidence}', got: {msg}"
+            );
+        }
+    }
+
+    /// A genuine not-found error and an empty listing must both still resolve to
+    /// TableNotFound (the local-FS and object-store representations of "missing").
+    #[tokio::test]
+    async fn test_table_resolution_missing_table_yields_table_not_found() {
+        for behavior in [ListBehavior::NotFound, ListBehavior::EmptyListing] {
+            let (namespace, _temp_dir, toggle) = failing_list_namespace().await;
+            *toggle.lock().unwrap() = Some(behavior);
+
+            let mut describe_req = DescribeTableRequest::new();
+            describe_req.id = Some(vec!["missing".to_string()]);
+            let err = namespace.describe_table(describe_req).await.unwrap_err();
+            assert_eq!(
+                mutation_error_code(err),
+                ErrorCode::TableNotFound,
+                "describe_table under {behavior:?} should be TableNotFound"
+            );
+
+            let mut exists_req = TableExistsRequest::new();
+            exists_req.id = Some(vec!["missing".to_string()]);
+            let err = namespace.table_exists(exists_req).await.unwrap_err();
+            assert_eq!(
+                mutation_error_code(err),
+                ErrorCode::TableNotFound,
+                "table_exists under {behavior:?} should be TableNotFound"
+            );
+        }
+    }
+
+    /// Hybrid (manifest + directory) resolution must exercise the
+    /// manifest→directory fall-through for a table that exists on disk but is not
+    /// registered in the manifest: the fall-through must succeed normally, and
+    /// must not degrade a storage error into TableNotFound.
+    ///
+    /// A `__manifest` table must actually exist for the manifest branch to run;
+    /// otherwise `manifest_ns_for_read()` is None and the manifest branch (and its
+    /// fall-through arm) is skipped entirely. We therefore create a *separate*
+    /// table through a manifest-enabled namespace first so `__manifest` exists.
+    #[tokio::test]
+    async fn test_hybrid_resolution_falls_through_and_does_not_mask_throttle() {
+        let temp_dir = TempStdDir::default();
+        let root_uri = file_object_store_uri(temp_dir.to_str().unwrap());
+        let behavior = Arc::new(Mutex::new(None));
+        let session = build_failing_list_session(behavior.clone());
+
+        // Seed table via a manifest-enabled namespace so `__manifest` exists.
+        let manifest_ns = DirectoryNamespaceBuilder::new(root_uri.clone())
+            .session(session.clone())
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        create_named_dir_table(&manifest_ns, "seed").await;
+
+        // The table under test: on disk but never registered in the manifest.
+        let dir_ns = DirectoryNamespaceBuilder::new(root_uri.clone())
+            .session(session.clone())
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        create_named_dir_table(&dir_ns, "checkpoint").await;
+
+        // Migration enabled so root-level reads consult the manifest and fall
+        // through to the directory check on a manifest miss.
+        let hybrid_ns = DirectoryNamespaceBuilder::new(root_uri)
+            .session(session)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .dir_listing_to_manifest_migration_enabled(true)
+            .build()
+            .await
+            .unwrap();
+
+        // (a) Healthy fall-through: the manifest reports "checkpoint" absent and it
+        // resolves via the directory listing (guards the migration lookup).
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["checkpoint".to_string()]);
+        hybrid_ns
+            .describe_table(describe_req)
+            .await
+            .expect("unregistered on-disk table should resolve via the manifest fall-through");
+
+        // (b) The throttle here surfaces from the directory check after the manifest
+        // reports absent; the fall-through arm's own storage-error guard is covered
+        // by the classify_storage_error / is_manifest_table_absent_error unit tests.
+        *behavior.lock().unwrap() = Some(ListBehavior::Throttle);
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["checkpoint".to_string()]);
+        let err = hybrid_ns.describe_table(describe_req).await.unwrap_err();
+        let code = mutation_error_code(err);
+        assert_ne!(
+            code,
+            ErrorCode::TableNotFound,
+            "hybrid resolution masked a throttle as TableNotFound"
+        );
+        assert!(
+            matches!(
+                code,
+                ErrorCode::Throttling | ErrorCode::ServiceUnavailable | ErrorCode::Internal
+            ),
+            "hybrid resolution should surface a storage error, got {code:?}"
+        );
+    }
+
+    #[test]
+    fn test_classify_storage_error_maps_variants_and_preserves_evidence() {
+        let throttle: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "list request failed, after 3 retries, max_retries: 3 - 503 ServerBusy".into(),
+        }
+        .into();
+        assert!(matches!(throttle, Error::IO { .. }));
+        let classified = DirectoryNamespace::classify_storage_error(throttle);
+        let msg = classified.to_string();
+        assert_eq!(mutation_error_code(classified), ErrorCode::Throttling);
+        assert!(
+            msg.to_ascii_lowercase().contains("serverbusy"),
+            "throttle evidence lost: {msg}"
+        );
+
+        let service: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "504 Gateway Timeout".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::classify_storage_error(service)),
+            ErrorCode::ServiceUnavailable
+        );
+
+        let internal: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "disk caught fire".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::classify_storage_error(internal)),
+            ErrorCode::Internal
+        );
+
+        // A pre-existing namespace error keeps its own code rather than being reclassified.
+        let preexisting: Error = NamespaceError::TableAlreadyExists {
+            message: "t".to_string(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::classify_storage_error(preexisting)),
+            ErrorCode::TableAlreadyExists
+        );
+    }
+
+    #[test]
+    fn test_is_manifest_table_absent_error() {
+        let table_not_found: Error = NamespaceError::TableNotFound {
+            message: "t".to_string(),
+        }
+        .into();
+        assert!(DirectoryNamespace::is_manifest_table_absent_error(
+            &table_not_found
+        ));
+        let raw_not_found: Error = ObjectStoreError::NotFound {
+            path: "t".to_string(),
+            source: "x".into(),
+        }
+        .into();
+        assert!(DirectoryNamespace::is_manifest_table_absent_error(
+            &raw_not_found
+        ));
+
+        let throttle: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "after 3 retries, max_retries: 3 ServerBusy".into(),
+        }
+        .into();
+        assert!(!DirectoryNamespace::is_manifest_table_absent_error(
+            &throttle
+        ));
+        let internal: Error = NamespaceError::Internal {
+            message: "boom".to_string(),
+        }
+        .into();
+        assert!(!DirectoryNamespace::is_manifest_table_absent_error(
+            &internal
+        ));
+    }
+
+    #[test]
+    fn test_map_open_error() {
+        let not_found = || NamespaceError::TableNotFound {
+            message: "table at 'x' not found: ...".to_string(),
+        };
+
+        let throttle: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(throttle, not_found())),
+            ErrorCode::Throttling
+        );
+
+        let generic_io: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "connection reset".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(generic_io, not_found())),
+            ErrorCode::Internal
+        );
+
+        let io_not_found: Error = ObjectStoreError::NotFound {
+            path: "x".to_string(),
+            source: "missing".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(
+                io_not_found,
+                not_found()
+            )),
+            ErrorCode::TableNotFound
+        );
+
+        let dataset_not_found = Error::dataset_not_found("x".to_string(), "missing".into());
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(
+                dataset_not_found,
+                not_found()
+            )),
+            ErrorCode::TableNotFound
+        );
+
+        // RefNotFound is not an IO error, so it is not reclassified as a storage error.
+        let ref_not_found = Error::RefNotFound {
+            message: "branch 'b' does not exist".to_string(),
+        };
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(
+                ref_not_found,
+                not_found()
+            )),
+            ErrorCode::TableNotFound
+        );
+
+        // The caller's not-found variant is honored, but a throttle still propagates.
+        let version_miss = Error::RefNotFound {
+            message: "version 5 does not exist".to_string(),
+        };
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(
+                version_miss,
+                NamespaceError::TableVersionNotFound {
+                    message: "version 5 not found".to_string(),
+                },
+            )),
+            ErrorCode::TableVersionNotFound
+        );
+        let version_throttle: Error = ObjectStoreError::Generic {
+            store: "test",
+            source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(),
+        }
+        .into();
+        assert_eq!(
+            mutation_error_code(DirectoryNamespace::map_open_error(
+                version_throttle,
+                NamespaceError::TableVersionNotFound {
+                    message: "version 5 not found".to_string(),
+                },
+            )),
+            ErrorCode::Throttling
+        );
+    }
+
     /// Helper to create test IPC data from a schema
     fn create_test_ipc_data(schema: &JsonArrowSchema) -> Vec<u8> {
         use arrow::ipc::writer::StreamWriter;
@@ -5574,6 +6473,43 @@ mod tests {
         create_ipc_data_from_batches(schema, vec![batch])
     }
 
+    async fn create_legacy_manifest_without_primary_key_metadata(root: &str) {
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use arrow::record_batch::{RecordBatch, RecordBatchIterator};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("object_id", DataType::Utf8, false),
+            Field::new("object_type", DataType::Utf8, false),
+            Field::new("location", DataType::Utf8, true),
+            Field::new("metadata", DataType::Utf8, true),
+            Field::new(
+                "base_objects",
+                DataType::List(Arc::new(Field::new("object_id", DataType::Utf8, true))),
+                true,
+            ),
+        ]));
+        let batch = RecordBatch::new_empty(schema.clone());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        Dataset::write(Box::new(reader), &format!("{}/__manifest", root), None)
+            .await
+            .unwrap();
+    }
+
+    async fn manifest_has_primary_key_metadata(root: &str) -> bool {
+        let dataset = Dataset::open(&format!("{}/__manifest", root))
+            .await
+            .unwrap();
+        dataset
+            .schema()
+            .field("object_id")
+            .map(|field| {
+                field
+                    .metadata
+                    .contains_key(lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+            })
+            .unwrap_or(false)
+    }
+
     fn create_vector_table_ipc_data() -> Vec<u8> {
         use arrow::array::{FixedSizeListArray, Float32Array, Int32Array};
         use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
@@ -9932,7 +10868,7 @@ mod tests {
             .unwrap();
 
         // Table status should show exists=true, is_deregistered=false
-        let status = namespace.check_table_status("test_table").await;
+        let status = namespace.check_table_status("test_table").await.unwrap();
         assert!(status.exists);
         assert!(!status.is_deregistered);
         assert!(!status.has_reserved_file);
@@ -12256,7 +13192,7 @@ mod tests {
             &plan_str,
             &[
                 "ProjectionExec: expr=[id@0 as id, name@2 as name",
-                "Take: columns=\"id, _rowid, (name)\"",
+                "projection=[name], source=stream(_rowid)",
                 "LanceRead: uri=",
                 "projection=[id]",
                 "row_id=true, row_addr=false",
@@ -12353,9 +13289,7 @@ mod tests {
                 "AnalyzeExec verbose=true",
                 "ProjectionExec: elapsed=",
                 "expr=[id@0 as id, name@2 as name",
-                "Take: elapsed=",
-                "columns=\"id, _rowid, (name)\"",
-                "CoalesceBatchesExec: elapsed=",
+                "projection=[name], source=stream(_rowid)",
                 "LanceRead: elapsed=",
                 "projection=[id]",
                 "row_id=true, row_addr=false",
@@ -12435,6 +13369,109 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_build_and_root_reads_do_not_create_manifest() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_path = std::path::Path::new(temp_path).join("__manifest");
+
+        let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        create_scalar_table(&dir_only_ns, "catalog").await;
+        assert!(!manifest_path.exists());
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        assert!(!manifest_path.exists());
+
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["catalog".to_string()]);
+        namespace.table_exists(exists_req).await.unwrap();
+        assert!(!manifest_path.exists());
+
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["catalog".to_string()]);
+        namespace.describe_table(describe_req).await.unwrap();
+        assert!(!manifest_path.exists());
+
+        let list_response = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(list_response.tables, vec!["catalog".to_string()]);
+        assert!(!manifest_path.exists());
+
+        let mut list_namespaces_req = ListNamespacesRequest::new();
+        list_namespaces_req.id = Some(vec!["workspace".to_string()]);
+        let err = namespace
+            .list_namespaces(list_namespaces_req)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let err = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec!["workspace".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut child_describe_req = DescribeTableRequest::new();
+        child_describe_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]);
+        let err = namespace
+            .describe_table(child_describe_req)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut child_exists_req = TableExistsRequest::new();
+        child_exists_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]);
+        let err = namespace.table_exists(child_exists_req).await.unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+        assert!(manifest_path.exists());
+    }
+
+    #[tokio::test]
+    async fn test_migrate_updates_read_opened_legacy_manifest() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        create_legacy_manifest_without_primary_key_metadata(temp_path).await;
+        assert!(!manifest_has_primary_key_metadata(temp_path).await);
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        assert!(!manifest_has_primary_key_metadata(temp_path).await);
+
+        let migrated = namespace.migrate().await.unwrap();
+        assert_eq!(migrated, 0);
+        assert!(manifest_has_primary_key_metadata(temp_path).await);
+    }
+
     #[tokio::test]
     async fn test_describe_declared_table_checks_versions_only_when_requested() {
         let temp_dir = TempStdDir::default();
diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs
index bca7408369e..1e52fc85bfe 100644
--- a/rust/lance-namespace-impls/src/dir/manifest.rs
+++ b/rust/lance-namespace-impls/src/dir/manifest.rs
@@ -36,7 +36,9 @@ use lance_index::progress::noop_progress;
 use lance_index::registry::IndexPluginRegistry;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
-use lance_index::scalar::{BuiltinIndexType, CreatedIndex, ScalarIndexParams};
+use lance_index::scalar::{
+    BuiltinIndexType, CreatedIndex, ScalarIndexParams, index_files_to_table,
+};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_io::stream::RecordBatchStream as LanceRecordBatchStream;
 use lance_namespace::LanceNamespace;
@@ -852,7 +854,60 @@ impl ManifestNamespace {
             Self::ensure_manifest_table_up_to_date(&root, &storage_options, session.clone())
                 .await?;
 
-        Ok(Self {
+        Ok(Self::new(
+            root,
+            storage_options,
+            session,
+            object_store,
+            base_path,
+            manifest_dataset,
+            dir_listing_enabled,
+            inline_optimization_enabled,
+            commit_retries,
+        ))
+    }
+
+    /// Open an existing manifest dataset without creating or migrating it.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn open_from_directory(
+        root: String,
+        storage_options: Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+        object_store: Arc<ObjectStore>,
+        base_path: Path,
+        dir_listing_enabled: bool,
+        inline_optimization_enabled: bool,
+        commit_retries: Option<u32>,
+    ) -> Result<Self> {
+        let manifest_dataset =
+            Self::open_manifest_table(&root, &storage_options, session.clone()).await?;
+
+        Ok(Self::new(
+            root,
+            storage_options,
+            session,
+            object_store,
+            base_path,
+            manifest_dataset,
+            dir_listing_enabled,
+            inline_optimization_enabled,
+            commit_retries,
+        ))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn new(
+        root: String,
+        storage_options: Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+        object_store: Arc<ObjectStore>,
+        base_path: Path,
+        manifest_dataset: DatasetConsistencyWrapper,
+        dir_listing_enabled: bool,
+        inline_optimization_enabled: bool,
+        commit_retries: Option<u32>,
+    ) -> Self {
+        Self {
             root,
             storage_options,
             session,
@@ -863,7 +918,7 @@ impl ManifestNamespace {
             inline_optimization_enabled,
             commit_retries,
             manifest_mutation_lock: Arc::new(Mutex::new(())),
-        })
+        }
     }
 
     /// Build object ID from namespace path and name
@@ -1226,7 +1281,7 @@ impl ManifestNamespace {
             index_version: trained_index.created_index.index_version as i32,
             created_at: None,
             base_id: None,
-            files: Some(trained_index.created_index.files),
+            files: Some(index_files_to_table(trained_index.created_index.files)),
         })
     }
 
@@ -2411,6 +2466,37 @@ impl ManifestNamespace {
         Ok(found_result)
     }
 
+    /// Load an existing manifest dataset without creating or migrating it.
+    async fn open_manifest_table(
+        root: &str,
+        storage_options: &Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+    ) -> Result<DatasetConsistencyWrapper> {
+        let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME);
+        log::debug!("Attempting to load manifest from {}", manifest_path);
+        let store_options = ObjectStoreParams {
+            storage_options_accessor: storage_options.as_ref().map(|opts| {
+                Arc::new(
+                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                        opts.clone(),
+                    ),
+                )
+            }),
+            ..Default::default()
+        };
+        let read_params = ReadParams {
+            session,
+            store_options: Some(store_options),
+            ..Default::default()
+        };
+        let dataset = DatasetBuilder::from_uri(&manifest_path)
+            .with_read_params(read_params)
+            .load()
+            .await?;
+        ensure_readable(dataset.metadata())?;
+        Ok(DatasetConsistencyWrapper::new(dataset))
+    }
+
     /// Create or load the manifest dataset, ensuring it has the latest schema setup.
     ///
     /// This function will:
@@ -2443,129 +2529,135 @@ impl ManifestNamespace {
             .with_read_params(read_params)
             .load()
             .await;
-        if let Ok(mut dataset) = dataset_result {
-            // Reject a manifest written with a reader feature flag this build
-            // does not understand before touching it.
-            ensure_readable(dataset.metadata())?;
-
-            // Check if the object_id field has primary key metadata, migrate if not
-            let needs_pk_migration = dataset
-                .schema()
-                .field("object_id")
-                .map(|f| {
-                    !f.metadata
-                        .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
-                })
-                .unwrap_or(false);
-
-            if needs_pk_migration {
-                // This legacy migration writes to the manifest, so confirm this
-                // build is allowed to write the current format first.
-                ensure_writable(dataset.metadata())?;
-                log::info!("Migrating __manifest table to add primary key metadata on object_id");
-                dataset
-                    .update_field_metadata()
-                    .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")])
-                    .map_err(|e| {
-                        lance_core::Error::from(NamespaceError::Internal {
-                            message: format!(
-                                "Failed to find object_id field for migration: {:?}",
-                                e
-                            ),
-                        })
-                    })?
-                    .await
-                    .map_err(|e| {
-                        lance_core::Error::from(NamespaceError::Internal {
-                            message: format!("Failed to migrate primary key metadata: {:?}", e),
-                        })
-                    })?;
-            }
-
-            Ok(DatasetConsistencyWrapper::new(dataset))
-        } else {
-            log::info!("Creating new manifest table at {}", manifest_path);
-            let schema = Self::manifest_schema();
-            let empty_batch = RecordBatch::new_empty(schema.clone());
-            let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone());
-
-            let store_params = ObjectStoreParams {
-                storage_options_accessor: storage_options.as_ref().map(|opts| {
-                    Arc::new(
-                        lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                            opts.clone(),
-                        ),
-                    )
-                }),
-                ..Default::default()
-            };
-            let write_params = WriteParams {
-                session: session.clone(),
-                store_params: Some(store_params),
-                ..Default::default()
-            };
-
-            let dataset =
-                Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await;
+        match dataset_result {
+            Ok(mut dataset) => {
+                // Reject a manifest written with a reader feature flag this build
+                // does not understand before touching it.
+                ensure_readable(dataset.metadata())?;
+
+                // Check if the object_id field has primary key metadata, migrate if not
+                let needs_pk_migration = dataset
+                    .schema()
+                    .field("object_id")
+                    .map(|f| {
+                        !f.metadata
+                            .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+                    })
+                    .unwrap_or(false);
 
-            // Handle race condition where another process created the manifest concurrently
-            match dataset {
-                Ok(dataset) => {
-                    log::info!(
-                        "Successfully created manifest table at {}, version={}, uri={}",
-                        manifest_path,
-                        dataset.version().version,
-                        dataset.uri()
-                    );
-                    Ok(DatasetConsistencyWrapper::new(dataset))
-                }
-                Err(ref e)
-                    if matches!(
-                        e,
-                        LanceError::DatasetAlreadyExists { .. }
-                            | LanceError::CommitConflict { .. }
-                            | LanceError::IncompatibleTransaction { .. }
-                            | LanceError::RetryableCommitConflict { .. }
-                    ) =>
-                {
-                    // Another process created the manifest concurrently, try to load it
+                if needs_pk_migration {
+                    // This legacy migration writes to the manifest, so confirm this
+                    // build is allowed to write the current format first.
+                    ensure_writable(dataset.metadata())?;
                     log::info!(
-                        "Manifest table was created by another process, loading it: {}",
-                        manifest_path
+                        "Migrating __manifest table to add primary key metadata on object_id"
                     );
-                    let recovery_store_options = ObjectStoreParams {
-                        storage_options_accessor: storage_options.as_ref().map(|opts| {
-                            Arc::new(
-                                lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                                    opts.clone(),
-                                ),
-                            )
-                        }),
-                        ..Default::default()
-                    };
-                    let recovery_read_params = ReadParams {
-                        session,
-                        store_options: Some(recovery_store_options),
-                        ..Default::default()
-                    };
-                    let dataset = DatasetBuilder::from_uri(&manifest_path)
-                        .with_read_params(recovery_read_params)
-                        .load()
-                        .await
+                    dataset
+                        .update_field_metadata()
+                        .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")])
                         .map_err(|e| {
                             lance_core::Error::from(NamespaceError::Internal {
                                 message: format!(
-                                    "Failed to load manifest dataset after creation conflict: {}",
+                                    "Failed to find object_id field for migration: {:?}",
                                     e
                                 ),
                             })
+                        })?
+                        .await
+                        .map_err(|e| {
+                            lance_core::Error::from(NamespaceError::Internal {
+                                message: format!("Failed to migrate primary key metadata: {:?}", e),
+                            })
                         })?;
-                    Ok(DatasetConsistencyWrapper::new(dataset))
                 }
-                Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
-                    message: format!("Failed to create manifest dataset: {:?}", e),
-                })),
+
+                Ok(DatasetConsistencyWrapper::new(dataset))
+            }
+            Err(err) if Self::is_not_found_load_error(&err) => {
+                log::info!("Creating new manifest table at {}", manifest_path);
+                let schema = Self::manifest_schema();
+                let empty_batch = RecordBatch::new_empty(schema.clone());
+                let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone());
+
+                let store_params = ObjectStoreParams {
+                    storage_options_accessor: storage_options.as_ref().map(|opts| {
+                        Arc::new(
+                            lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                                opts.clone(),
+                            ),
+                        )
+                    }),
+                    ..Default::default()
+                };
+                let write_params = WriteParams {
+                    session: session.clone(),
+                    store_params: Some(store_params),
+                    ..Default::default()
+                };
+
+                let dataset =
+                    Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await;
+
+                // Handle race condition where another process created the manifest concurrently
+                match dataset {
+                    Ok(dataset) => {
+                        log::info!(
+                            "Successfully created manifest table at {}, version={}, uri={}",
+                            manifest_path,
+                            dataset.version().version,
+                            dataset.uri()
+                        );
+                        Ok(DatasetConsistencyWrapper::new(dataset))
+                    }
+                    Err(ref e)
+                        if matches!(
+                            e,
+                            LanceError::DatasetAlreadyExists { .. }
+                                | LanceError::CommitConflict { .. }
+                                | LanceError::IncompatibleTransaction { .. }
+                                | LanceError::RetryableCommitConflict { .. }
+                        ) =>
+                    {
+                        // Another process created the manifest concurrently, try to load it
+                        log::info!(
+                            "Manifest table was created by another process, loading it: {}",
+                            manifest_path
+                        );
+                        let recovery_store_options = ObjectStoreParams {
+                            storage_options_accessor: storage_options.as_ref().map(|opts| {
+                                Arc::new(
+                                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                                        opts.clone(),
+                                    ),
+                                )
+                            }),
+                            ..Default::default()
+                        };
+                        let recovery_read_params = ReadParams {
+                            session,
+                            store_options: Some(recovery_store_options),
+                            ..Default::default()
+                        };
+                        let dataset = DatasetBuilder::from_uri(&manifest_path)
+                            .with_read_params(recovery_read_params)
+                            .load()
+                            .await
+                            .map_err(|e| {
+                                lance_core::Error::from(NamespaceError::Internal {
+                                    message: format!(
+                                        "Failed to load manifest dataset after creation conflict: {}",
+                                        e
+                                    ),
+                                })
+                            })?;
+                        Ok(DatasetConsistencyWrapper::new(dataset))
+                    }
+                    Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create manifest dataset: {:?}", e),
+                    })),
+                }
             }
+            Err(err) => Err(err),
         }
     }
 
diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs
index 78b7e352207..81bd57c1a22 100644
--- a/rust/lance-table/benches/manifest_intern.rs
+++ b/rust/lance-table/benches/manifest_intern.rs
@@ -59,6 +59,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec<pb::DataFragment>
                 file_size_bytes: 0,
                 base_id: None,
             }],
+            overlays: vec![],
             deletion_file: None,
             row_id_sequence: None,
             physical_rows: 1000,
@@ -135,6 +136,7 @@ fn make_diverse_pb_fragments(
                     file_size_bytes: 0,
                     base_id: None,
                 }],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_sequence: None,
                 physical_rows: 1000,
diff --git a/rust/lance-table/src/feature_flags.rs b/rust/lance-table/src/feature_flags.rs
index 096f0da79e5..41b8e415f8e 100644
--- a/rust/lance-table/src/feature_flags.rs
+++ b/rust/lance-table/src/feature_flags.rs
@@ -20,8 +20,22 @@ pub const FLAG_TABLE_CONFIG: u64 = 8;
 pub const FLAG_BASE_PATHS: u64 = 16;
 /// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest
 pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32;
+/// Fragments contain data overlay files, which supply new values for a subset of
+/// cells without rewriting base data files. A reader that does not understand
+/// overlays must refuse the dataset, since ignoring an overlay would silently
+/// return stale base values.
+///
+/// Data overlay files are not yet a released feature: in release builds this flag
+/// is treated as unknown (so a release reader/writer refuses an overlay dataset)
+/// unless [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set, which lets benchmarks opt in.
+/// Debug builds always understand it so tests exercise the path.
+pub const FLAG_UNSTABLE_DATA_OVERLAY_FILES: u64 = 64;
 /// The first bit that is unknown as a feature flag
-pub const FLAG_UNKNOWN: u64 = 64;
+pub const FLAG_UNKNOWN: u64 = 128;
+
+/// Environment variable that opts a release build into reading and writing data
+/// overlay files before the feature is generally released.
+pub const ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV: &str = "LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES";
 
 /// Set the reader and writer feature flags in the manifest based on the contents of the manifest.
 pub fn apply_feature_flags(
@@ -71,18 +85,62 @@ pub fn apply_feature_flags(
         manifest.writer_feature_flags |= FLAG_BASE_PATHS;
     }
 
+    // Overlay files change cell values on read, so a reader that ignores them
+    // would return stale base values. Both readers and writers must understand
+    // them.
+    let has_overlays = manifest
+        .fragments
+        .iter()
+        .any(|frag| !frag.overlays.is_empty());
+    if has_overlays {
+        manifest.reader_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
+        manifest.writer_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
+    }
+
     if disable_transaction_file {
         manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE;
     }
     Ok(())
 }
 
+/// Whether this build understands data overlay files: always in debug builds,
+/// and in release builds only when [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set.
+fn data_overlay_files_enabled() -> bool {
+    cfg!(debug_assertions) || std::env::var_os(ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV).is_some()
+}
+
+/// Clear `flag` from `flags` when its gating feature is not enabled in this
+/// build; leave it set otherwise. One call per unstable flag, so support for
+/// several unstable features chains cleanly.
+fn mark_supported(flags: &mut u64, flag: u64, feature_enabled: bool) {
+    if !feature_enabled {
+        *flags &= !flag;
+    }
+}
+
+/// The feature-flag bits this build understands, given whether overlay support
+/// is enabled. Split out from [`supported_flags`] so the policy is testable
+/// without toggling the build profile or environment.
+fn supported_flags_when(overlay_enabled: bool) -> u64 {
+    let mut supported = FLAG_UNKNOWN - 1;
+    mark_supported(
+        &mut supported,
+        FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+        overlay_enabled,
+    );
+    supported
+}
+
+fn supported_flags() -> u64 {
+    supported_flags_when(data_overlay_files_enabled())
+}
+
 pub fn can_read_dataset(reader_flags: u64) -> bool {
-    reader_flags < FLAG_UNKNOWN
+    reader_flags & !supported_flags() == 0
 }
 
 pub fn can_write_dataset(writer_flags: u64) -> bool {
-    writer_flags < FLAG_UNKNOWN
+    writer_flags & !supported_flags() == 0
 }
 
 pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool {
@@ -103,6 +161,13 @@ mod tests {
         assert!(can_read_dataset(super::FLAG_TABLE_CONFIG));
         assert!(can_read_dataset(super::FLAG_BASE_PATHS));
         assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
+        // Overlay support is gated on the build profile / env opt-in, so the
+        // flag is readable exactly when overlays are enabled (see
+        // test_data_overlay_flag_release_gating for the full policy).
+        assert_eq!(
+            can_read_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
+            data_overlay_files_enabled()
+        );
         assert!(can_read_dataset(
             super::FLAG_DELETION_FILES
                 | super::FLAG_STABLE_ROW_IDS
@@ -111,6 +176,58 @@ mod tests {
         assert!(!can_read_dataset(super::FLAG_UNKNOWN));
     }
 
+    #[test]
+    fn test_data_overlay_flag_release_gating() {
+        // Release default (overlays disabled): the overlay flag is treated as
+        // unknown so the dataset is refused, while other known flags still pass.
+        let supported = supported_flags_when(false);
+        assert_eq!(supported & FLAG_UNSTABLE_DATA_OVERLAY_FILES, 0);
+        assert_eq!(FLAG_DELETION_FILES & !supported, 0);
+        assert_ne!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
+        // Enabled (debug or env opt-in): the overlay flag is understood.
+        let supported = supported_flags_when(true);
+        assert_eq!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
+    }
+
+    #[test]
+    fn test_apply_feature_flags_sets_overlay_flag() {
+        use crate::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use crate::format::{DataFile, DataStorageFormat, Fragment};
+        use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
+        use lance_core::datatypes::Schema;
+        use roaring::RoaringBitmap;
+        use std::collections::HashMap;
+        use std::sync::Arc;
+
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            arrow_schema::DataType::Int64,
+            false,
+        )]);
+        let schema = Schema::try_from(&arrow_schema).unwrap();
+        let mut fragment = Fragment::new(0);
+        fragment.overlays = vec![DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![0], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: 1,
+        }];
+        let mut manifest = Manifest::new(
+            schema,
+            Arc::new(vec![fragment]),
+            DataStorageFormat::default(),
+            HashMap::new(),
+        );
+        apply_feature_flags(&mut manifest, false, false).unwrap();
+        assert_ne!(
+            manifest.reader_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+            0
+        );
+        assert_ne!(
+            manifest.writer_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+            0
+        );
+    }
+
     #[test]
     fn test_write_check() {
         assert!(can_write_dataset(0));
@@ -120,6 +237,13 @@ mod tests {
         assert!(can_write_dataset(super::FLAG_TABLE_CONFIG));
         assert!(can_write_dataset(super::FLAG_BASE_PATHS));
         assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
+        // Overlay support is gated on the build profile / env opt-in, so the
+        // flag is writable exactly when overlays are enabled (see
+        // test_data_overlay_flag_release_gating for the full policy).
+        assert_eq!(
+            can_write_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
+            data_overlay_files_enabled()
+        );
         assert!(can_write_dataset(
             super::FLAG_DELETION_FILES
                 | super::FLAG_STABLE_ROW_IDS
diff --git a/rust/lance-table/src/format.rs b/rust/lance-table/src/format.rs
index 842c76f1e58..5a5db7919d3 100644
--- a/rust/lance-table/src/format.rs
+++ b/rust/lance-table/src/format.rs
@@ -7,6 +7,7 @@ use uuid::Uuid;
 mod fragment;
 mod index;
 mod manifest;
+pub mod overlay;
 mod transaction;
 
 pub use crate::rowids::version::{
diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs
index 431e466dbd4..4929f54b7ff 100644
--- a/rust/lance-table/src/format/fragment.rs
+++ b/rust/lance-table/src/format/fragment.rs
@@ -13,6 +13,7 @@ use lance_io::utils::CachedFileSize;
 use object_store::path::Path;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
+use super::overlay::{DataOverlayFile, sort_overlays_newest_last};
 use crate::format::pb;
 
 use crate::rowids::version::{
@@ -375,6 +376,15 @@ impl DataFileFieldInterner {
                 .into_iter()
                 .map(|f| self.intern_data_file(f))
                 .collect::<Result<_>>()?,
+            overlays: {
+                let mut overlays = p
+                    .overlays
+                    .into_iter()
+                    .map(DataOverlayFile::try_from)
+                    .collect::<Result<Vec<_>>>()?;
+                sort_overlays_newest_last(&mut overlays);
+                overlays
+            },
             deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?,
             row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?,
             physical_rows,
@@ -483,6 +493,12 @@ pub struct Fragment {
     /// Files within the fragment.
     pub files: Vec<DataFile>,
 
+    /// Overlay files supplying new values for a subset of cells without
+    /// rewriting the base data files. Order is significant: a later entry is
+    /// newer than an earlier one. See [`DataOverlayFile`] for resolution rules.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub overlays: Vec<DataOverlayFile>,
+
     /// Optional file with deleted local row offsets.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub deletion_file: Option<DeletionFile>,
@@ -510,6 +526,7 @@ impl Fragment {
         Self {
             id,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: None,
@@ -549,6 +566,7 @@ impl Fragment {
         Self {
             id,
             files: vec![DataFile::new_legacy(path, schema, None, None)],
+            overlays: vec![],
             deletion_file: None,
             physical_rows,
             row_id_meta: None,
@@ -669,6 +687,15 @@ impl TryFrom<pb::DataFragment> for Fragment {
                 .into_iter()
                 .map(DataFile::try_from)
                 .collect::<Result<_>>()?,
+            overlays: {
+                let mut overlays = p
+                    .overlays
+                    .into_iter()
+                    .map(DataOverlayFile::try_from)
+                    .collect::<Result<Vec<_>>>()?;
+                sort_overlays_newest_last(&mut overlays);
+                overlays
+            },
             deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?,
             row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?,
             physical_rows,
@@ -716,6 +743,7 @@ impl From<&Fragment> for pb::DataFragment {
         Self {
             id: f.id,
             files: f.files.iter().map(pb::DataFile::from).collect(),
+            overlays: f.overlays.iter().map(pb::DataOverlayFile::from).collect(),
             deletion_file,
             row_id_sequence,
             physical_rows: f.physical_rows.unwrap_or_default() as u64,
@@ -728,12 +756,108 @@ impl From<&Fragment> for pb::DataFragment {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::format::overlay::OverlayCoverage;
     use arrow_schema::{
         DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema,
     };
     use object_store::path::Path;
+    use roaring::RoaringBitmap;
     use serde_json::{Value, json};
 
+    #[test]
+    fn test_data_overlay_roundtrip() {
+        // A fragment carrying a dense overlay round-trips through protobuf and
+        // back, and the parsed coverage bitmap is recovered per field.
+        let mut bitmap = RoaringBitmap::new();
+        bitmap.insert(1);
+        bitmap.insert(3);
+
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(bitmap.clone()),
+            committed_version: 7,
+        };
+        let mut fragment = Fragment::new(0);
+        fragment.files = vec![DataFile::new_legacy_from_fields(
+            "base.lance",
+            vec![1, 3],
+            None,
+        )];
+        fragment.overlays = vec![overlay];
+
+        let proto = pb::DataFragment::from(&fragment);
+        assert_eq!(proto.overlays.len(), 1);
+        let round_tripped = Fragment::try_from(proto).unwrap();
+        assert_eq!(round_tripped, fragment);
+
+        // Dense coverage applies to every field.
+        let recovered = round_tripped.overlays[0].coverage_for_field(0).unwrap();
+        assert_eq!(*recovered, bitmap);
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(5).unwrap(),
+            bitmap
+        );
+    }
+
+    #[test]
+    fn test_data_overlay_sparse_per_field_coverage() {
+        // A sparse overlay carries one bitmap per field, recovered by position.
+        let name_coverage = RoaringBitmap::from_iter([2u32, 3]);
+        let embedding_coverage = RoaringBitmap::from_iter([1u32]);
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-1.lance", vec![2, 4], None),
+            coverage: OverlayCoverage::sparse(vec![
+                name_coverage.clone(),
+                embedding_coverage.clone(),
+            ]),
+            committed_version: 3,
+        };
+        let mut fragment = Fragment::new(1);
+        fragment.overlays = vec![overlay];
+
+        let round_tripped = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap();
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(0).unwrap(),
+            name_coverage
+        );
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(1).unwrap(),
+            embedding_coverage
+        );
+    }
+
+    #[test]
+    fn test_overlays_sorted_newest_last_on_load() {
+        // Overlays load stable-sorted by committed_version (newest last), with
+        // list position preserved as the tiebreak for equal versions.
+        let mk = |version: u64, field: i32| DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: version,
+        };
+        let mut fragment = Fragment::new(0);
+        // Written out of order: v5, v2, v2 (second), v3.
+        fragment.overlays = vec![mk(5, 1), mk(2, 2), mk(2, 3), mk(3, 4)];
+
+        let loaded = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap();
+        let versions: Vec<u64> = loaded
+            .overlays
+            .iter()
+            .map(|o| o.committed_version)
+            .collect();
+        assert_eq!(versions, vec![2, 2, 3, 5]);
+        // Stable: the two v2 overlays keep their original relative order (field 2
+        // before field 3).
+        assert_eq!(
+            loaded.overlays[0].data_file.fields.as_ref(),
+            [2i32].as_slice()
+        );
+        assert_eq!(
+            loaded.overlays[1].data_file.fields.as_ref(),
+            [3i32].as_slice()
+        );
+    }
+
     #[test]
     fn test_new_fragment() {
         let path = "foobar.lance";
diff --git a/rust/lance-table/src/format/manifest.rs b/rust/lance-table/src/format/manifest.rs
index 9845061b7e4..83c9643e600 100644
--- a/rust/lance-table/src/format/manifest.rs
+++ b/rust/lance-table/src/format/manifest.rs
@@ -900,7 +900,7 @@ impl TryFrom<pb::Manifest> for Manifest {
             Some(format) => DataStorageFormat::from(format),
         };
 
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)?;
 
         Ok(Self {
             schema,
@@ -1316,6 +1316,7 @@ mod tests {
                     vec![0, 1, 2],
                     None,
                 )],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1328,6 +1329,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 43], None),
                     DataFile::new_legacy_from_fields("path3", vec![2], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
diff --git a/rust/lance-table/src/format/overlay.rs b/rust/lance-table/src/format/overlay.rs
new file mode 100644
index 00000000000..5e729411502
--- /dev/null
+++ b/rust/lance-table/src/format/overlay.rs
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Data overlay files.
+//!
+//! An overlay file supplies new values for a subset of `(physical offset, field)`
+//! cells within a fragment, without rewriting the fragment's base data files. See
+//! the Data Overlay Files specification for the full rules; the invariants this
+//! module relies on are:
+//!
+//! - **Physical-offset coverage.** Coverage bitmaps index *physical* row offsets
+//!   (positions in the base data files, counting deleted rows), so they are stable
+//!   across deletions, like deletion vectors.
+//! - **Rank-based values.** The overlay's `data_file` stores one value column per
+//!   field, with no row-offset key column. Within a value column, a covered
+//!   offset's value sits at its **rank** — the 0-based count of set bits below it
+//!   in that field's coverage bitmap.
+//! - **Dense vs. sparse coverage.** A dense overlay shares one bitmap across every
+//!   field ([`OverlayCoverage::Shared`]); a sparse overlay carries one bitmap per
+//!   field ([`OverlayCoverage::PerField`]).
+//! - **Parse once.** Bitmaps are parsed from their 32-bit Roaring encoding a single
+//!   time when the fragment loads and held behind an `Arc`, so cloning a fragment
+//!   is cheap.
+//! - **Newest-last ordering.** A fragment's overlays are stored newest-last and
+//!   stable-sorted by `committed_version` on load (see [`sort_overlays_newest_last`]),
+//!   with list position breaking ties for equal versions. When two overlays cover
+//!   the same `(offset, field)`, the higher `committed_version` wins.
+//! - **Field tombstones.** When new base values are written for a field (a
+//!   DataReplacement, or an in-place column rewrite), any overlay value for that
+//!   field is stale and must stop shadowing the fresh base. The field is marked
+//!   obsolete in the overlay's `data_file.fields` with [`TOMBSTONE_FIELD_ID`]
+//!   (the same sentinel used for obsolete base columns) rather than physically
+//!   removed, so the overlay's other fields — and its coverage positions — stay
+//!   intact (see [`tombstone_overlay_fields`]).
+
+use std::sync::Arc;
+
+use lance_core::Error;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::error::Result;
+use roaring::RoaringBitmap;
+use serde::{Deserialize, Serialize};
+
+use object_store::path::Path;
+
+use super::DataFile;
+use crate::format::pb;
+
+/// Field-id sentinel marking a tombstoned (obsolete) field within an overlay's
+/// `data_file.fields`. Matches the tombstone convention for obsolete columns in
+/// base data files; a tombstoned field's values are ignored on read.
+pub const TOMBSTONE_FIELD_ID: i32 = -2;
+
+/// Which `(physical offset, field)` cells a [`DataOverlayFile`] provides values
+/// for.
+///
+/// Bitmaps are parsed from their 32-bit Roaring encoding once when the fragment
+/// is loaded and held behind an `Arc` so cloning a fragment is cheap; use
+/// [`DataOverlayFile::coverage_for_field`] to obtain the one that applies to a
+/// given field.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(into = "OverlayCoverageBytes", try_from = "OverlayCoverageBytes")]
+pub enum OverlayCoverage {
+    /// A single bitmap that applies to every field in the overlay's
+    /// `data_file.fields` (a dense / rectangular overlay): every covered offset
+    /// has a value for every field.
+    Shared(Arc<RoaringBitmap>),
+    /// One bitmap per field, in the same order as the overlay's
+    /// `data_file.fields` (a sparse overlay): different fields may cover
+    /// different offset sets.
+    PerField(Vec<Arc<RoaringBitmap>>),
+}
+
+/// Serialized form of [`OverlayCoverage`] — each bitmap as its 32-bit Roaring
+/// byte encoding. The in-memory form parses these once at load.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+enum OverlayCoverageBytes {
+    Shared(Vec<u8>),
+    PerField(Vec<Vec<u8>>),
+}
+
+// The bytes come from a persisted overlay (the protobuf manifest or a
+// serialized fragment), so a decode failure is on-disk corruption, not caller
+// input. `path` locates the overlay's data file when known (empty on the serde
+// path, which deserializes coverage in isolation).
+fn deserialize_roaring(bytes: &[u8], path: &Path) -> Result<RoaringBitmap> {
+    RoaringBitmap::deserialize_from(bytes).map_err(|e| {
+        Error::corrupt_file(
+            path.clone(),
+            format!("failed to deserialize overlay coverage bitmap: {e}"),
+        )
+    })
+}
+
+fn serialize_roaring(bitmap: &RoaringBitmap) -> Vec<u8> {
+    let mut bytes = Vec::with_capacity(bitmap.serialized_size());
+    // Writing to a Vec is infallible.
+    bitmap.serialize_into(&mut bytes).unwrap();
+    bytes
+}
+
+impl From<OverlayCoverage> for OverlayCoverageBytes {
+    fn from(coverage: OverlayCoverage) -> Self {
+        match coverage {
+            OverlayCoverage::Shared(bitmap) => Self::Shared(serialize_roaring(&bitmap)),
+            OverlayCoverage::PerField(bitmaps) => {
+                Self::PerField(bitmaps.iter().map(|b| serialize_roaring(b)).collect())
+            }
+        }
+    }
+}
+
+impl TryFrom<OverlayCoverageBytes> for OverlayCoverage {
+    type Error = Error;
+
+    fn try_from(bytes: OverlayCoverageBytes) -> Result<Self> {
+        // Serde deserializes the coverage in isolation, so the owning data
+        // file's path is not available here.
+        let path = Path::default();
+        Ok(match bytes {
+            OverlayCoverageBytes::Shared(b) => {
+                Self::Shared(Arc::new(deserialize_roaring(&b, &path)?))
+            }
+            OverlayCoverageBytes::PerField(bs) => Self::PerField(
+                bs.iter()
+                    .map(|b| deserialize_roaring(b, &path).map(Arc::new))
+                    .collect::<Result<_>>()?,
+            ),
+        })
+    }
+}
+
+impl DeepSizeOf for OverlayCoverage {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // The same `Arc<RoaringBitmap>` is shared across every clone of a
+        // fragment, so mark each Arc's pointer and count its heap only the first
+        // time it is seen — otherwise walking many fragments double-counts the
+        // shared bitmaps. RoaringBitmap does not expose its allocation size; its
+        // serialized size is a cheap, close proxy for the heap it holds.
+        let bitmap_heap = |bitmap: &Arc<RoaringBitmap>,
+                           context: &mut lance_core::deepsize::Context| {
+            if context.mark_seen(Arc::as_ptr(bitmap) as usize) {
+                std::mem::size_of::<RoaringBitmap>() + bitmap.serialized_size()
+            } else {
+                0
+            }
+        };
+        match self {
+            Self::Shared(bitmap) => bitmap_heap(bitmap, context),
+            Self::PerField(bitmaps) => {
+                bitmaps.capacity() * std::mem::size_of::<Arc<RoaringBitmap>>()
+                    + bitmaps
+                        .iter()
+                        .map(|b| bitmap_heap(b, context))
+                        .sum::<usize>()
+            }
+        }
+    }
+}
+
+impl OverlayCoverage {
+    /// Build a dense coverage from a single bitmap shared across every field.
+    pub fn dense(bitmap: RoaringBitmap) -> Self {
+        Self::Shared(Arc::new(bitmap))
+    }
+
+    /// Build a sparse coverage from one bitmap per field.
+    pub fn sparse(bitmaps: Vec<RoaringBitmap>) -> Self {
+        Self::PerField(bitmaps.into_iter().map(Arc::new).collect())
+    }
+}
+
+/// An overlay file supplies new values for a subset of `(physical offset, field)`
+/// cells within a fragment, without rewriting the fragment's base data files. See
+/// the [module documentation](self) for the coverage, rank, and versioning rules.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct DataOverlayFile {
+    /// The data file storing the overlay's new cell values.
+    pub data_file: DataFile,
+    /// Which cells this overlay provides values for.
+    pub coverage: OverlayCoverage,
+    /// The dataset version at which this overlay became effective (the version of
+    /// the commit that introduced it, stamped at commit time and re-stamped on
+    /// retry). Higher wins when two overlays cover the same `(offset, field)`.
+    pub committed_version: u64,
+}
+
+impl DataOverlayFile {
+    /// The parsed coverage bitmap that applies to the field stored at
+    /// `field_pos` within `data_file.fields`.
+    ///
+    /// For a dense overlay the same shared bitmap is returned for every field;
+    /// for a sparse overlay the per-field bitmap at `field_pos` is returned. The
+    /// bitmap is already parsed, so this is a cheap `Arc` clone.
+    pub fn coverage_for_field(&self, field_pos: usize) -> Result<Arc<RoaringBitmap>> {
+        match &self.coverage {
+            OverlayCoverage::Shared(bitmap) => Ok(bitmap.clone()),
+            OverlayCoverage::PerField(bitmaps) => {
+                bitmaps.get(field_pos).cloned().ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "overlay per-field coverage has {} bitmaps but field position {} was requested",
+                        bitmaps.len(),
+                        field_pos
+                    ))
+                })
+            }
+        }
+    }
+}
+
+/// Stable-sort a fragment's overlays newest-last by `committed_version`. The
+/// stable sort preserves list position as the tiebreak for equal versions, so
+/// resolution can rely on the ordering without re-checking. See the [module
+/// documentation](self) for the ordering invariant.
+pub fn sort_overlays_newest_last(overlays: &mut [DataOverlayFile]) {
+    overlays.sort_by_key(|overlay| overlay.committed_version);
+}
+
+/// Verify a fragment's overlays are stored newest-last (non-decreasing
+/// `committed_version`), the ordering invariant readers rely on for
+/// resolution. Returns an error identifying the first out-of-order pair.
+///
+/// [`sort_overlays_newest_last`] normalizes on load; this is the write-side
+/// guard that rejects any commit path that assembled overlays out of order. See
+/// the [module documentation](self) for the ordering invariant.
+pub fn verify_overlays_newest_last(overlays: &[DataOverlayFile]) -> Result<()> {
+    for pair in overlays.windows(2) {
+        if pair[0].committed_version > pair[1].committed_version {
+            return Err(Error::invalid_input(format!(
+                "overlay files must be stored newest-last, but committed_version {} precedes {}",
+                pair[0].committed_version, pair[1].committed_version
+            )));
+        }
+    }
+    Ok(())
+}
+
+/// Tombstone `fields` across a fragment's `overlays`, dropping any overlay left
+/// with no live fields.
+///
+/// Called when new base values are written for those fields (a DataReplacement,
+/// or an in-place column rewrite): the stale overlay values must stop shadowing
+/// the fresh base. Each matching field id is replaced with [`TOMBSTONE_FIELD_ID`]
+/// in place, preserving the overlay's remaining fields and its coverage positions
+/// (a per-field coverage bitmap stays aligned with `data_file.fields`). An overlay
+/// whose fields are now all tombstoned is removed entirely. See the [module
+/// documentation](self) for the tombstone invariant.
+pub fn tombstone_overlay_fields(overlays: &mut Vec<DataOverlayFile>, fields: &[u32]) {
+    for overlay in overlays.iter_mut() {
+        let tombstoned: Vec<i32> = overlay
+            .data_file
+            .fields
+            .iter()
+            .map(|&field| {
+                if field >= 0 && fields.contains(&(field as u32)) {
+                    TOMBSTONE_FIELD_ID
+                } else {
+                    field
+                }
+            })
+            .collect();
+        overlay.data_file.fields = tombstoned.into();
+    }
+    overlays.retain(|overlay| {
+        overlay
+            .data_file
+            .fields
+            .iter()
+            .any(|&field| field != TOMBSTONE_FIELD_ID)
+    });
+}
+
+impl From<&DataOverlayFile> for pb::DataOverlayFile {
+    fn from(overlay: &DataOverlayFile) -> Self {
+        let coverage = match &overlay.coverage {
+            OverlayCoverage::Shared(bitmap) => {
+                pb::data_overlay_file::Coverage::SharedOffsetBitmap(serialize_roaring(bitmap))
+            }
+            OverlayCoverage::PerField(bitmaps) => {
+                pb::data_overlay_file::Coverage::FieldCoverage(pb::FieldCoverage {
+                    offset_bitmaps: bitmaps.iter().map(|b| serialize_roaring(b)).collect(),
+                })
+            }
+        };
+        Self {
+            data_file: Some(pb::DataFile::from(&overlay.data_file)),
+            coverage: Some(coverage),
+            committed_version: overlay.committed_version,
+        }
+    }
+}
+
+impl TryFrom<pb::DataOverlayFile> for DataOverlayFile {
+    type Error = Error;
+
+    fn try_from(proto: pb::DataOverlayFile) -> Result<Self> {
+        let data_file = proto
+            .data_file
+            .ok_or_else(|| Error::invalid_input("DataOverlayFile is missing its data_file"))?;
+        let path = Path::from(data_file.path.as_str());
+        let coverage = match proto.coverage {
+            Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(bytes)) => {
+                OverlayCoverage::Shared(Arc::new(deserialize_roaring(&bytes, &path)?))
+            }
+            Some(pb::data_overlay_file::Coverage::FieldCoverage(fc)) => OverlayCoverage::PerField(
+                fc.offset_bitmaps
+                    .iter()
+                    .map(|b| deserialize_roaring(b, &path).map(Arc::new))
+                    .collect::<Result<_>>()?,
+            ),
+            None => {
+                return Err(Error::invalid_input(
+                    "DataOverlayFile is missing its coverage",
+                ));
+            }
+        };
+        Ok(Self {
+            data_file: DataFile::try_from(data_file)?,
+            coverage,
+            committed_version: proto.committed_version,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_data_overlay_missing_fields_error() {
+        // A DataOverlayFile proto missing its coverage or data_file is rejected.
+        let no_coverage = pb::DataOverlayFile {
+            data_file: Some(pb::DataFile::from(&DataFile::new_legacy_from_fields(
+                "overlay.lance",
+                vec![3],
+                None,
+            ))),
+            coverage: None,
+            committed_version: 1,
+        };
+        let err = DataOverlayFile::try_from(no_coverage).unwrap_err();
+        assert!(err.to_string().contains("missing its coverage"), "{err}");
+
+        let no_data_file = pb::DataOverlayFile {
+            data_file: None,
+            coverage: Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(
+                serialize_roaring(&RoaringBitmap::from_iter([0u32])),
+            )),
+            committed_version: 1,
+        };
+        let err = DataOverlayFile::try_from(no_data_file).unwrap_err();
+        assert!(err.to_string().contains("missing its data_file"), "{err}");
+    }
+
+    #[test]
+    fn test_overlay_coverage_serde_json_roundtrip() {
+        // The custom serde impl round-trips through JSON for dense/sparse,
+        // including empty bitmaps and a zero-bitmap sparse coverage.
+        for coverage in [
+            OverlayCoverage::dense(RoaringBitmap::from_iter([1u32, 5, 100])),
+            OverlayCoverage::dense(RoaringBitmap::new()),
+            OverlayCoverage::sparse(vec![
+                RoaringBitmap::from_iter([2u32, 3]),
+                RoaringBitmap::new(),
+            ]),
+            OverlayCoverage::sparse(vec![]),
+        ] {
+            let json = serde_json::to_string(&coverage).unwrap();
+            let back: OverlayCoverage = serde_json::from_str(&json).unwrap();
+            assert_eq!(back, coverage);
+        }
+    }
+
+    #[test]
+    fn test_tombstone_overlay_fields() {
+        // An overlay covering fields [3, 5]: replacing field 5 tombstones just
+        // field 5's slot and keeps field 3. An overlay covering only field 5 is
+        // dropped entirely. An overlay touching no replaced field is untouched.
+        let mut overlays = vec![
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("a.lance", vec![3, 5], None),
+                coverage: OverlayCoverage::sparse(vec![
+                    RoaringBitmap::from_iter([0u32]),
+                    RoaringBitmap::from_iter([1u32]),
+                ]),
+                committed_version: 1,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("b.lance", vec![5], None),
+                coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                committed_version: 1,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("c.lance", vec![7], None),
+                coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                committed_version: 1,
+            },
+        ];
+
+        tombstone_overlay_fields(&mut overlays, &[5]);
+
+        // The single-field overlay on field 5 is gone; the others remain.
+        assert_eq!(overlays.len(), 2);
+        // Field 3 preserved, field 5 tombstoned in place (coverage stays aligned).
+        assert_eq!(
+            overlays[0].data_file.fields.as_ref(),
+            &[3, TOMBSTONE_FIELD_ID]
+        );
+        // The untouched overlay keeps its field.
+        assert_eq!(overlays[1].data_file.fields.as_ref(), &[7]);
+    }
+
+    #[test]
+    fn test_verify_overlays_newest_last() {
+        let mk = |version: u64| DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: version,
+        };
+        // Non-decreasing (including equal versions) is accepted.
+        assert!(verify_overlays_newest_last(&[]).is_ok());
+        assert!(verify_overlays_newest_last(&[mk(1), mk(2), mk(2), mk(5)]).is_ok());
+        // A newer version before an older one is rejected.
+        let err = verify_overlays_newest_last(&[mk(2), mk(1)]).unwrap_err();
+        assert!(err.to_string().contains("newest-last"), "{err}");
+    }
+
+    #[test]
+    fn test_coverage_for_field_out_of_bounds() {
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None),
+            coverage: OverlayCoverage::sparse(vec![
+                RoaringBitmap::from_iter([1u32]),
+                RoaringBitmap::from_iter([2u32]),
+            ]),
+            committed_version: 1,
+        };
+        assert!(overlay.coverage_for_field(0).is_ok());
+        assert!(overlay.coverage_for_field(1).is_ok());
+        let err = overlay.coverage_for_field(5).unwrap_err();
+        assert!(err.to_string().contains("field position"), "{err}");
+    }
+}
diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs
index e1a4086730b..d682bf3a4a2 100644
--- a/rust/lance-table/src/io/commit.rs
+++ b/rust/lance-table/src/io/commit.rs
@@ -1091,7 +1091,7 @@ pub async fn commit_handler_from_url(
 
     match url.scheme() {
         "file" | "file-object-store" => Ok(local_handler),
-        "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "shared-memory" => {
+        "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "tos" | "shared-memory" => {
             Ok(Arc::new(ConditionalPutCommitHandler))
         }
         #[cfg(not(feature = "dynamodb"))]
@@ -1966,19 +1966,27 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_commit_handler_from_url_memory_schemes() {
-        // Both `memory://` and `shared-memory://` must route to
-        // ConditionalPutCommitHandler — otherwise concurrent writers fall
-        // through to UnsafeCommitHandler and silently clobber each other's
-        // manifests.
-        for url in ["memory://bucket-a/ds", "shared-memory://bucket-a/ds"] {
-            let handler = commit_handler_from_url(url, &None).await.unwrap();
-            assert_eq!(
-                format!("{:?}", handler),
-                "ConditionalPutCommitHandler",
-                "{url} should route to ConditionalPutCommitHandler",
-            );
-        }
+    #[rstest::rstest]
+    #[case::memory("memory://bucket-a/ds")]
+    #[case::shared_memory("shared-memory://bucket-a/ds")]
+    #[case::s3("s3://bucket-a/ds")]
+    #[case::gs("gs://bucket-a/ds")]
+    #[case::az("az://bucket-a/ds")]
+    #[case::abfss("abfss://bucket-a/ds")]
+    #[case::oss("oss://bucket-a/ds")]
+    #[case::cos("cos://bucket-a/ds")]
+    #[case::tos("tos://bucket-a/ds")]
+    async fn test_commit_handler_from_url_conditional_put_schemes(#[case] url: &str) {
+        // Every scheme whose store supports atomic put-if-not-exists must
+        // route to ConditionalPutCommitHandler — otherwise concurrent writers
+        // fall through to UnsafeCommitHandler and silently clobber each
+        // other's manifests.
+        let handler = commit_handler_from_url(url, &None).await.unwrap();
+        assert_eq!(
+            format!("{:?}", handler),
+            "ConditionalPutCommitHandler",
+            "{url} should route to ConditionalPutCommitHandler",
+        );
     }
 
     /// A [CommitLock] whose lease records whether it was released, so we can
diff --git a/rust/lance-table/src/io/commit/external_manifest.rs b/rust/lance-table/src/io/commit/external_manifest.rs
index 22ebaa10b4a..c039066844f 100644
--- a/rust/lance-table/src/io/commit/external_manifest.rs
+++ b/rust/lance-table/src/io/commit/external_manifest.rs
@@ -134,20 +134,18 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync {
             info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_path.as_ref());
         }
 
-        // Get final e_tag (may change after copy for large files)
-        let e_tag = if copied && size < 5 * 1024 * 1024 {
-            e_tag
-        } else {
-            let meta = object_store.head(&final_path).await?;
-            meta.e_tag
-        };
+        // A copy creates a new object whose metadata may differ from the source.
+        // Read the destination metadata before publishing the final path.
+        let final_meta = object_store.head(&final_path).await?;
+        let final_size = final_meta.size;
+        let final_e_tag = final_meta.e_tag;
 
         let location = ManifestLocation {
             version,
             path: final_path.clone(),
-            size: Some(size),
+            size: Some(final_size),
             naming_scheme,
-            e_tag: e_tag.clone(),
+            e_tag: final_e_tag.clone(),
         };
 
         if !copied {
@@ -159,8 +157,8 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync {
             base_path.as_ref(),
             version,
             final_path.as_ref(),
-            size,
-            e_tag,
+            final_size,
+            final_e_tag,
         )
         .await?;
 
@@ -347,6 +345,69 @@ pub struct ExternalManifestCommitHandler {
 }
 
 impl ExternalManifestCommitHandler {
+    async fn verify_finalized_manifest_location(
+        &self,
+        base_path: &Path,
+        location: ManifestLocation,
+        object_store: &dyn OSObjectStore,
+    ) -> std::result::Result<ManifestLocation, Error> {
+        match object_store.head(&location.path).await {
+            Ok(ObjectMeta { size, e_tag, .. }) => {
+                let ManifestLocation {
+                    version,
+                    path,
+                    size: expected_size,
+                    naming_scheme,
+                    e_tag: expected_e_tag,
+                } = location;
+
+                let size = match expected_size {
+                    Some(expected_size) if expected_size != size => {
+                        return Err(Error::corrupt_file(
+                            path,
+                            format!(
+                                "Manifest size mismatch for version {}: external store expected {}, object store returned {}",
+                                version, expected_size, size
+                            ),
+                        ));
+                    }
+                    Some(expected_size) => Some(expected_size),
+                    None => Some(size),
+                };
+
+                let e_tag = match expected_e_tag {
+                    Some(expected_e_tag) => {
+                        if e_tag.as_ref() != Some(&expected_e_tag) {
+                            return Err(Error::corrupt_file(
+                                path,
+                                format!(
+                                    "Manifest e_tag mismatch for version {}: external store expected {:?}, object store returned {:?}",
+                                    version, expected_e_tag, e_tag
+                                ),
+                            ));
+                        }
+                        Some(expected_e_tag)
+                    }
+                    None => e_tag,
+                };
+
+                Ok(ManifestLocation {
+                    version,
+                    path,
+                    size,
+                    naming_scheme,
+                    e_tag,
+                })
+            }
+            Err(ObjectStoreError::NotFound { .. }) => {
+                // The external store may hold a stale finalized V2 path while
+                // the object store still has the manifest at the V1 location.
+                default_resolve_version(base_path, location.version, object_store).await
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+
     /// The manifest is considered committed once the staging manifest is written
     /// to object store and that path is committed to the external store.
     ///
@@ -363,7 +424,6 @@ impl ExternalManifestCommitHandler {
         staging_manifest_path: &Path,
         version: u64,
         size: u64,
-        e_tag: Option<String>,
         store: &dyn OSObjectStore,
         naming_scheme: ManifestNamingScheme,
     ) -> std::result::Result<ManifestLocation, Error> {
@@ -380,25 +440,18 @@ impl ExternalManifestCommitHandler {
             info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_manifest_path.as_ref());
         }
 
-        // On S3, the etag can change if originally was MultipartUpload and later was Copy
-        // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html#AmazonS3-Type-Object-ETag
-        // We only do MultipartUpload for > 5MB files, so we can skip this check
-        // if size < 5MB. However, we need to double check the final_manifest_path
-        // exists before we change the external store, otherwise we may point to a
-        // non-existing manifest.
-        let e_tag = if copied && size < 5 * 1024 * 1024 {
-            e_tag
-        } else {
-            let meta = store.head(&final_manifest_path).await?;
-            meta.e_tag
-        };
+        // A copy creates a new object whose metadata may differ from the source.
+        // Read the destination metadata before publishing the final path.
+        let final_meta = store.head(&final_manifest_path).await?;
+        let final_size = final_meta.size;
+        let final_e_tag = final_meta.e_tag;
 
         let location = ManifestLocation {
             version,
             path: final_manifest_path,
-            size: Some(size),
+            size: Some(final_size),
             naming_scheme,
-            e_tag,
+            e_tag: final_e_tag,
         };
 
         if !copied {
@@ -411,7 +464,7 @@ impl ExternalManifestCommitHandler {
                 base_path.as_ref(),
                 version,
                 location.path.as_ref(),
-                size,
+                final_size,
                 location.e_tag.clone(),
             )
             .await?;
@@ -441,29 +494,30 @@ impl CommitHandler for ExternalManifestCommitHandler {
             .await?;
 
         match location {
-            Some(ManifestLocation {
-                version,
-                path,
-                size,
-                naming_scheme,
-                e_tag,
-            }) => {
-                // The path is finalized, no need to check object store
-                if path.extension() == Some(MANIFEST_EXTENSION) {
-                    return Ok(ManifestLocation {
-                        version,
-                        path,
-                        size,
-                        naming_scheme,
-                        e_tag,
-                    });
+            Some(location) => {
+                if location.path.extension() == Some(MANIFEST_EXTENSION) {
+                    return self
+                        .verify_finalized_manifest_location(
+                            base_path,
+                            location,
+                            object_store.inner.as_ref(),
+                        )
+                        .await;
                 }
 
-                let (size, e_tag) = if let Some(size) = size {
-                    (size, e_tag)
+                let ManifestLocation {
+                    version,
+                    path,
+                    size,
+                    naming_scheme,
+                    e_tag: _,
+                } = location;
+
+                let size = if let Some(size) = size {
+                    size
                 } else {
                     match object_store.inner.head(&path).await {
-                        Ok(meta) => (meta.size, meta.e_tag),
+                        Ok(meta) => meta.size,
                         Err(ObjectStoreError::NotFound { .. }) => {
                             // there may be other threads that have finished executing finalize_manifest.
                             let new_location = self
@@ -482,7 +536,6 @@ impl CommitHandler for ExternalManifestCommitHandler {
                         &path,
                         version,
                         size,
-                        e_tag.clone(),
                         &object_store.inner,
                         naming_scheme,
                     )
@@ -552,19 +605,20 @@ impl CommitHandler for ExternalManifestCommitHandler {
             Err(e) => return Err(e),
         };
 
-        // finalized path, just return
         if location.path.extension() == Some(MANIFEST_EXTENSION) {
-            return Ok(location);
+            return self
+                .verify_finalized_manifest_location(base_path, location, object_store)
+                .await;
         }
 
         let naming_scheme =
             ManifestNamingScheme::detect_scheme_staging(location.path.filename().unwrap());
 
-        let (size, e_tag) = if let Some(size) = location.size {
-            (size, location.e_tag.clone())
+        let size = if let Some(size) = location.size {
+            size
         } else {
             let meta = object_store.head(&location.path).await?;
-            (meta.size as u64, meta.e_tag)
+            meta.size
         };
 
         self.finalize_manifest(
@@ -572,7 +626,6 @@ impl CommitHandler for ExternalManifestCommitHandler {
             &location.path,
             version,
             size,
-            e_tag,
             object_store,
             naming_scheme,
         )
diff --git a/rust/lance-table/src/rowids.rs b/rust/lance-table/src/rowids.rs
index ab5dac72b48..6cdc907cda5 100644
--- a/rust/lance-table/src/rowids.rs
+++ b/rust/lance-table/src/rowids.rs
@@ -355,11 +355,7 @@ impl RowIdSequence {
             while (index - rows_passed) >= cur_seg_len {
                 rows_passed += cur_seg_len;
                 cur_seg = seg_iter.next();
-                if let Some(cur_seg) = cur_seg {
-                    cur_seg_len = cur_seg.len();
-                } else {
-                    return None;
-                }
+                cur_seg_len = cur_seg?.len();
             }
 
             Some(cur_seg.unwrap().get(index - rows_passed).unwrap())
diff --git a/rust/lance-table/src/system_index/mem_wal.rs b/rust/lance-table/src/system_index/mem_wal.rs
index 1d82fd9e44f..d3f5a157e94 100644
--- a/rust/lance-table/src/system_index/mem_wal.rs
+++ b/rust/lance-table/src/system_index/mem_wal.rs
@@ -15,45 +15,45 @@ pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal";
 /// Type alias for shard identifier (UUID v4).
 pub type ShardId = Uuid;
 
-/// A flushed MemTable generation and its storage location.
+/// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FlushedGeneration {
+pub struct SsTable {
     pub generation: u64,
     pub path: String,
 }
 
-impl From<&FlushedGeneration> for pb::FlushedGeneration {
-    fn from(fg: &FlushedGeneration) -> Self {
+impl From<&SsTable> for pb::SsTable {
+    fn from(sstable: &SsTable) -> Self {
         Self {
-            generation: fg.generation,
-            path: fg.path.clone(),
+            generation: sstable.generation,
+            path: sstable.path.clone(),
         }
     }
 }
 
-impl From<pb::FlushedGeneration> for FlushedGeneration {
-    fn from(fg: pb::FlushedGeneration) -> Self {
+impl From<pb::SsTable> for SsTable {
+    fn from(sstable: pb::SsTable) -> Self {
         Self {
-            generation: fg.generation,
-            path: fg.path,
+            generation: sstable.generation,
+            path: sstable.path,
         }
     }
 }
 
-/// A shard's merged generation, used in MemWalIndexDetails.
+/// A pointer to the latest SSTable compacted for a shard.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
-pub struct MergedGeneration {
+pub struct CompactedSsTable {
     pub shard_id: Uuid,
     pub generation: u64,
 }
 
-impl DeepSizeOf for MergedGeneration {
+impl DeepSizeOf for CompactedSsTable {
     fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         0 // UUID is 16 bytes fixed size, no heap allocations
     }
 }
 
-impl MergedGeneration {
+impl CompactedSsTable {
     pub fn new(shard_id: Uuid, generation: u64) -> Self {
         Self {
             shard_id,
@@ -62,41 +62,41 @@ impl MergedGeneration {
     }
 }
 
-impl From<&MergedGeneration> for pb::MergedGeneration {
-    fn from(mg: &MergedGeneration) -> Self {
+impl From<&CompactedSsTable> for pb::CompactedSsTable {
+    fn from(sstable: &CompactedSsTable) -> Self {
         Self {
-            shard_id: Some((&mg.shard_id).into()),
-            generation: mg.generation,
+            shard_id: Some((&sstable.shard_id).into()),
+            generation: sstable.generation,
         }
     }
 }
 
-impl TryFrom<pb::MergedGeneration> for MergedGeneration {
+impl TryFrom<pb::CompactedSsTable> for CompactedSsTable {
     type Error = Error;
 
-    fn try_from(mg: pb::MergedGeneration) -> lance_core::Result<Self> {
-        let shard_id = mg
+    fn try_from(sstable: pb::CompactedSsTable) -> lance_core::Result<Self> {
+        let shard_id = sstable
             .shard_id
             .as_ref()
             .map(Uuid::try_from)
-            .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??;
+            .ok_or_else(|| Error::invalid_input("Missing shard_id in CompactedSsTable"))??;
         Ok(Self {
             shard_id,
-            generation: mg.generation,
+            generation: sstable.generation,
         })
     }
 }
 
-/// Tracks which merged generation a base table index has been rebuilt to cover.
-/// Used to determine whether to read from flushed MemTable indexes or base table.
+/// Tracks which compacted SSTable generation a base table index covers.
+/// Used to determine whether to read from SSTable indexes or base table.
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
 pub struct IndexCatchupProgress {
     pub index_name: String,
-    pub caught_up_generations: Vec<MergedGeneration>,
+    pub caught_up_generations: Vec<CompactedSsTable>,
 }
 
 impl IndexCatchupProgress {
-    pub fn new(index_name: String, caught_up_generations: Vec<MergedGeneration>) -> Self {
+    pub fn new(index_name: String, caught_up_generations: Vec<CompactedSsTable>) -> Self {
         Self {
             index_name,
             caught_up_generations,
@@ -108,8 +108,8 @@ impl IndexCatchupProgress {
     pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
         self.caught_up_generations
             .iter()
-            .find(|mg| &mg.shard_id == shard_id)
-            .map(|mg| mg.generation)
+            .find(|sstable| &sstable.shard_id == shard_id)
+            .map(|sstable| sstable.generation)
     }
 }
 
@@ -120,7 +120,7 @@ impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress {
             caught_up_generations: icp
                 .caught_up_generations
                 .iter()
-                .map(|mg| mg.into())
+                .map(|sstable| sstable.into())
                 .collect(),
         }
     }
@@ -135,7 +135,7 @@ impl TryFrom<pb::IndexCatchupProgress> for IndexCatchupProgress {
             caught_up_generations: icp
                 .caught_up_generations
                 .into_iter()
-                .map(MergedGeneration::try_from)
+                .map(CompactedSsTable::try_from)
                 .collect::<lance_core::Result<_>>()?,
         })
     }
@@ -198,7 +198,7 @@ pub struct ShardManifest {
     /// 1-based.
     pub wal_entry_position_last_seen: u64,
     pub current_generation: u64,
-    pub flushed_generations: Vec<FlushedGeneration>,
+    pub sstables: Vec<SsTable>,
     /// Lifecycle status (drop-table 2PC). Defaults to `Active`; preserved
     /// across claims via `..base` so only fresh constructions set it.
     pub status: ShardStatus,
@@ -207,7 +207,7 @@ pub struct ShardManifest {
 impl DeepSizeOf for ShardManifest {
     fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.shard_field_values.deep_size_of_children(context)
-            + self.flushed_generations.deep_size_of_children(context)
+            + self.sstables.deep_size_of_children(context)
     }
 }
 
@@ -229,7 +229,7 @@ impl From<&ShardManifest> for pb::ShardManifest {
             replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
             wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
             current_generation: rm.current_generation,
-            flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(),
+            sstables: rm.sstables.iter().map(|sstable| sstable.into()).collect(),
             status: rm.status.to_i32(),
         }
     }
@@ -258,11 +258,7 @@ impl TryFrom<pb::ShardManifest> for ShardManifest {
             replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
             wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
             current_generation: rm.current_generation,
-            flushed_generations: rm
-                .flushed_generations
-                .into_iter()
-                .map(FlushedGeneration::from)
-                .collect(),
+            sstables: rm.sstables.into_iter().map(SsTable::from).collect(),
             status: ShardStatus::from_i32(rm.status),
         })
     }
@@ -338,7 +334,7 @@ pub struct MemWalIndexDetails {
     pub inline_snapshots: Option<Vec<u8>>,
     pub sharding_specs: Vec<ShardingSpec>,
     pub maintained_indexes: Vec<String>,
-    pub merged_generations: Vec<MergedGeneration>,
+    pub compacted_sstables: Vec<CompactedSsTable>,
     pub index_catchup: Vec<IndexCatchupProgress>,
     /// Default `ShardWriter` configuration values for this MemWAL index.
     ///
@@ -357,10 +353,10 @@ impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails {
             inline_snapshots: details.inline_snapshots.clone(),
             sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(),
             maintained_indexes: details.maintained_indexes.clone(),
-            merged_generations: details
-                .merged_generations
+            compacted_sstables: details
+                .compacted_sstables
                 .iter()
-                .map(|mg| mg.into())
+                .map(|sstable| sstable.into())
                 .collect(),
             index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(),
             writer_config_defaults: details.writer_config_defaults.clone(),
@@ -382,10 +378,10 @@ impl TryFrom<pb::MemWalIndexDetails> for MemWalIndexDetails {
                 .map(ShardingSpec::from)
                 .collect(),
             maintained_indexes: details.maintained_indexes,
-            merged_generations: details
-                .merged_generations
+            compacted_sstables: details
+                .compacted_sstables
                 .into_iter()
-                .map(MergedGeneration::try_from)
+                .map(CompactedSsTable::try_from)
                 .collect::<lance_core::Result<_>>()?,
             index_catchup: details
                 .index_catchup
@@ -408,12 +404,12 @@ impl MemWalIndex {
         Self { details }
     }
 
-    pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
+    pub fn compacted_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
         self.details
-            .merged_generations
+            .compacted_sstables
             .iter()
-            .find(|mg| &mg.shard_id == shard_id)
-            .map(|mg| mg.generation)
+            .find(|sstable| &sstable.shard_id == shard_id)
+            .map(|sstable| sstable.generation)
     }
 
     /// Get the caught up generation for a specific index and shard.
@@ -427,12 +423,12 @@ impl MemWalIndex {
     }
 
     /// Check if an index is fully caught up for a shard.
-    /// Returns true if the index covers all merged data for the shard.
+    /// Returns true if the index covers all compacted data for the shard.
     pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool {
-        let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0);
+        let compacted_gen = self.compacted_generation_for_shard(shard_id).unwrap_or(0);
         let caught_up_gen = self.index_caught_up_generation(index_name, shard_id);
 
         // If not tracked in index_catchup, assumed fully caught up
-        caught_up_gen.is_none_or(|generation| generation >= merged_gen)
+        caught_up_gen.is_none_or(|generation| generation >= compacted_gen)
     }
 }
diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs
new file mode 100644
index 00000000000..0d00aa2b94c
--- /dev/null
+++ b/rust/lance-tokenizer/src/code_tokenizer.rs
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{iter::Peekable, str::CharIndices};
+
+use crate::{Token, TokenStream, Tokenizer};
+
+/// Tokenizer for code-like text.
+///
+/// Identifiers are Unicode alphanumeric characters plus `_`. Other characters
+/// are lexical boundaries. When operator indexing is enabled, recognized
+/// multi-character operators use longest-match tokenization and remaining
+/// operator characters are emitted individually.
+///
+/// # Examples
+///
+/// ```
+/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream};
+///
+/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(true)).build();
+/// let mut stream = analyzer.token_stream("a::b");
+///
+/// assert!(stream.advance());
+/// assert_eq!(stream.token().text, "a");
+/// assert!(stream.advance());
+/// assert_eq!(stream.token().text, "::");
+/// ```
+#[derive(Clone, Default)]
+pub struct CodeLexTokenizer {
+    index_operators: bool,
+    token: Token,
+}
+
+impl CodeLexTokenizer {
+    pub fn new(index_operators: bool) -> Self {
+        Self {
+            index_operators,
+            token: Token::default(),
+        }
+    }
+}
+
+/// Token stream produced by [`CodeLexTokenizer`].
+pub struct CodeLexTokenStream<'a> {
+    text: &'a str,
+    chars: Peekable<CharIndices<'a>>,
+    token: &'a mut Token,
+    index_operators: bool,
+}
+
+impl Tokenizer for CodeLexTokenizer {
+    type TokenStream<'a> = CodeLexTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        CodeLexTokenStream {
+            text,
+            chars: text.char_indices().peekable(),
+            token: &mut self.token,
+            index_operators: self.index_operators,
+        }
+    }
+}
+
+fn is_identifier_char(ch: char) -> bool {
+    ch == '_' || ch.is_alphanumeric()
+}
+
+fn is_operator_char(ch: char) -> bool {
+    matches!(
+        ch,
+        '!' | '%' | '&' | '*' | '+' | '-' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '|' | '~'
+    )
+}
+
+const MULTI_CHAR_OPERATORS: &[&str] = &[
+    ">>>=", "<<=", ">>=", "&&=", "||=", "??=", "**=", "//=", "===", "!==", ">>>", "<=>", "::",
+    "->", "=>", "==", "!=", "<=", ">=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "&=",
+    "|=", "^=", "<<", ">>", "**", "//", "??", ":=", "<-", "|>", "~=",
+];
+
+impl CodeLexTokenStream<'_> {
+    fn search_token_end(&mut self, predicate: impl Fn(char) -> bool) -> usize {
+        while let Some((_, ch)) = self.chars.peek() {
+            if !predicate(*ch) {
+                break;
+            }
+            self.chars.next();
+        }
+        self.chars
+            .peek()
+            .map(|(offset, _)| *offset)
+            .unwrap_or(self.text.len())
+    }
+
+    fn operator_token_end(&mut self, offset_from: usize) -> usize {
+        let remaining = &self.text[offset_from..];
+        let operator_len = MULTI_CHAR_OPERATORS
+            .iter()
+            .filter(|operator| remaining.starts_with(**operator))
+            .map(|operator| operator.len())
+            .max()
+            .unwrap_or(1);
+        let token_end = offset_from + operator_len;
+        while self
+            .chars
+            .peek()
+            .is_some_and(|(offset, _)| *offset < token_end)
+        {
+            self.chars.next();
+        }
+        token_end
+    }
+}
+
+impl TokenStream for CodeLexTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        self.token.text.clear();
+        while let Some((offset_from, ch)) = self.chars.next() {
+            let token_end = if is_identifier_char(ch) {
+                self.search_token_end(is_identifier_char)
+            } else if self.index_operators && is_operator_char(ch) {
+                self.operator_token_end(offset_from)
+            } else {
+                continue;
+            };
+
+            self.token.position = self.token.position.wrapping_add(1);
+            self.token.position_length = 1;
+            self.token.offset_from = offset_from;
+            self.token.offset_to = token_end;
+            self.token.text.push_str(&self.text[offset_from..token_end]);
+            return true;
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{CodeLexTokenizer, TextAnalyzer, Token};
+
+    fn collect_tokens(text: &str, index_operators: bool) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(index_operators)).build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_code_lex_tokenizer_identifiers() {
+        let tokens = collect_tokens("std::vector user-name parse.HTML2JSON", false);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            texts,
+            vec!["std", "vector", "user", "name", "parse", "HTML2JSON"]
+        );
+    }
+
+    #[test]
+    fn test_code_lex_tokenizer_operators() {
+        let tokens = collect_tokens("a::b != c->d", true);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(texts, vec!["a", "::", "b", "!=", "c", "->", "d"]);
+    }
+
+    #[test]
+    fn test_code_lex_tokenizer_splits_adjacent_operators() {
+        let tokens = collect_tokens("value.parse::<usize>()", true);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(texts, vec!["value", "parse", "::", "<", "usize", ">"]);
+    }
+}
diff --git a/rust/lance-tokenizer/src/lib.rs b/rust/lance-tokenizer/src/lib.rs
index 0708022b384..fc1d9512cfd 100644
--- a/rust/lance-tokenizer/src/lib.rs
+++ b/rust/lance-tokenizer/src/lib.rs
@@ -4,6 +4,7 @@
 mod alphanum_only;
 mod analyzer;
 mod ascii_folding_filter;
+mod code_tokenizer;
 mod icu;
 #[cfg(feature = "tokenizer-jieba")]
 mod jieba;
@@ -16,6 +17,7 @@ mod stemmer;
 mod stop_word_filter;
 mod tokenizer_api;
 mod whitespace_tokenizer;
+mod word_delimiter_filter;
 
 #[cfg(feature = "tokenizer-lindera")]
 mod lindera;
@@ -23,6 +25,7 @@ mod lindera;
 pub use alphanum_only::AlphaNumOnlyFilter;
 pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder};
 pub use ascii_folding_filter::AsciiFoldingFilter;
+pub use code_tokenizer::CodeLexTokenizer;
 pub use icu::IcuTokenizer;
 #[cfg(feature = "tokenizer-jieba")]
 pub use jieba::JiebaTokenizer;
@@ -37,3 +40,4 @@ pub use stemmer::{Language, Stemmer};
 pub use stop_word_filter::StopWordFilter;
 pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer};
 pub use whitespace_tokenizer::WhitespaceTokenizer;
+pub use word_delimiter_filter::WordDelimiterFilter;
diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs
index 2acf0b3dbd5..9a690b0ec06 100644
--- a/rust/lance-tokenizer/src/stop_word_filter.rs
+++ b/rust/lance-tokenizer/src/stop_word_filter.rs
@@ -17,7 +17,12 @@ fn all_stop_words() -> impl Iterator<Item = &'static str> {
         stop_words::get("ar"),
         stopwords::DANISH,
         stopwords::DUTCH,
-        stopwords::ENGLISH,
+        // Use the fuller `stop-words` crate English list (~198 words) rather
+        // than the local Tantivy-style list (~33 words), which omits extremely
+        // common pronouns/function words (you, my, your, we, she, what, ...).
+        // Those omissions let the highest-frequency English tokens through the
+        // ICU stop-word path and build pathologically large posting lists.
+        stop_words::get("en"),
         stopwords::FINNISH,
         stopwords::FRENCH,
         stopwords::GERMAN,
@@ -51,7 +56,11 @@ impl StopWordFilter {
             Language::Arabic => stop_words::get("ar"),
             Language::Danish => stopwords::DANISH,
             Language::Dutch => stopwords::DUTCH,
-            Language::English => stopwords::ENGLISH,
+            // Use the fuller `stop-words` crate English list (~198 words); the
+            // local Tantivy-style list (~33 words) omits common pronouns/function
+            // words (you, my, your, we, ...) that would otherwise leak through
+            // stop-word removal and build pathologically large posting lists.
+            Language::English => stop_words::get("en"),
             Language::Finnish => stopwords::FINNISH,
             Language::French => stopwords::FRENCH,
             Language::German => stopwords::GERMAN,
diff --git a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs
index 227556ba527..2ac3f4a28aa 100644
--- a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs
+++ b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs
@@ -37,12 +37,6 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
-pub const ENGLISH: &[&str] = &[
-    "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it",
-    "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these",
-    "they", "this", "to", "was", "will", "with",
-];
-
 pub const DANISH: &[&str] = &[
     "og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som", "på", "de", "med", "han", "af",
     "for", "ikke", "der", "var", "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde",
diff --git a/rust/lance-tokenizer/src/word_delimiter_filter.rs b/rust/lance-tokenizer/src/word_delimiter_filter.rs
new file mode 100644
index 00000000000..70138529efe
--- /dev/null
+++ b/rust/lance-tokenizer/src/word_delimiter_filter.rs
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::VecDeque;
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+/// Splits code identifiers into subword tokens.
+///
+/// The original identifier can be preserved at the first subword position. If
+/// a token is not a compound identifier then it is passed through unchanged.
+///
+/// # Examples
+///
+/// ```
+/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream, WordDelimiterFilter};
+///
+/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false))
+///     .filter(WordDelimiterFilter::new(true, true))
+///     .build();
+/// let mut stream = analyzer.token_stream("parseHTML2JSON");
+///
+/// let mut tokens = Vec::new();
+/// stream.process(&mut |token| tokens.push(token.text.clone()));
+/// assert_eq!(tokens, vec!["parseHTML2JSON", "parse", "HTML", "2", "JSON"]);
+/// ```
+#[derive(Clone)]
+pub struct WordDelimiterFilter {
+    preserve_original: bool,
+    split_on_numerics: bool,
+}
+
+impl WordDelimiterFilter {
+    pub fn new(preserve_original: bool, split_on_numerics: bool) -> Self {
+        Self {
+            preserve_original,
+            split_on_numerics,
+        }
+    }
+}
+
+impl TokenFilter for WordDelimiterFilter {
+    type Tokenizer<T: Tokenizer> = WordDelimiterFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        WordDelimiterFilterWrapper {
+            tokenizer,
+            preserve_original: self.preserve_original,
+            split_on_numerics: self.split_on_numerics,
+        }
+    }
+}
+
+#[derive(Clone)]
+/// Tokenizer wrapper produced by [`WordDelimiterFilter`].
+pub struct WordDelimiterFilterWrapper<T> {
+    tokenizer: T,
+    preserve_original: bool,
+    split_on_numerics: bool,
+}
+
+impl<T: Tokenizer> Tokenizer for WordDelimiterFilterWrapper<T> {
+    type TokenStream<'a> = WordDelimiterTokenStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        WordDelimiterTokenStream {
+            tail: self.tokenizer.token_stream(text),
+            current: Token::default(),
+            pending: VecDeque::new(),
+            preserve_original: self.preserve_original,
+            split_on_numerics: self.split_on_numerics,
+            position_offset: 0,
+        }
+    }
+}
+
+/// Token stream produced by [`WordDelimiterFilterWrapper`].
+pub struct WordDelimiterTokenStream<T> {
+    tail: T,
+    current: Token,
+    pending: VecDeque<Token>,
+    preserve_original: bool,
+    split_on_numerics: bool,
+    position_offset: usize,
+}
+
+#[derive(Clone, Copy)]
+struct CharInfo {
+    offset: usize,
+    ch: char,
+}
+
+fn is_identifier_text(text: &str) -> bool {
+    text.chars().any(|ch| ch == '_' || ch.is_alphanumeric())
+}
+
+fn split_identifier(text: &str, split_on_numerics: bool) -> Vec<(usize, usize)> {
+    let mut pieces = Vec::new();
+    let mut segment_start = None;
+    for (offset, ch) in text.char_indices() {
+        if ch == '_' {
+            if let Some(start) = segment_start.take() {
+                split_segment(text, start, offset, split_on_numerics, &mut pieces);
+            }
+        } else if segment_start.is_none() {
+            segment_start = Some(offset);
+        }
+    }
+    if let Some(start) = segment_start {
+        split_segment(text, start, text.len(), split_on_numerics, &mut pieces);
+    }
+    pieces
+}
+
+fn split_segment(
+    text: &str,
+    start: usize,
+    end: usize,
+    split_on_numerics: bool,
+    pieces: &mut Vec<(usize, usize)>,
+) {
+    if start == end {
+        return;
+    }
+    let chars = text[start..end]
+        .char_indices()
+        .map(|(offset, ch)| CharInfo {
+            offset: start + offset,
+            ch,
+        })
+        .collect::<Vec<_>>();
+    let mut piece_start = start;
+    for idx in 1..chars.len() {
+        if is_boundary(&chars, idx, split_on_numerics) {
+            pieces.push((piece_start, chars[idx].offset));
+            piece_start = chars[idx].offset;
+        }
+    }
+    pieces.push((piece_start, end));
+}
+
+fn is_boundary(chars: &[CharInfo], idx: usize, split_on_numerics: bool) -> bool {
+    let prev = chars[idx - 1].ch;
+    let cur = chars[idx].ch;
+    if split_on_numerics && prev.is_ascii_digit() != cur.is_ascii_digit() {
+        return true;
+    }
+    if prev.is_lowercase() && cur.is_uppercase() {
+        return true;
+    }
+    if prev.is_uppercase()
+        && cur.is_uppercase()
+        && chars
+            .get(idx + 1)
+            .is_some_and(|next| next.ch.is_lowercase())
+    {
+        return true;
+    }
+    false
+}
+
+impl<T: TokenStream> WordDelimiterTokenStream<T> {
+    fn refill(&mut self) -> bool {
+        while self.pending.is_empty() {
+            if !self.tail.advance() {
+                return false;
+            }
+            self.enqueue_current_tail_token();
+        }
+        true
+    }
+
+    fn enqueue_current_tail_token(&mut self) {
+        let token = self.tail.token();
+        let adjusted_position = token.position.saturating_add(self.position_offset);
+        if !is_identifier_text(&token.text) {
+            let mut token = token.clone();
+            token.position = adjusted_position;
+            self.pending.push_back(token);
+            return;
+        }
+
+        let parts = split_identifier(&token.text, self.split_on_numerics);
+        if parts.len() <= 1 {
+            let mut token = token.clone();
+            token.position = adjusted_position;
+            self.pending.push_back(token);
+            return;
+        }
+
+        if self.preserve_original {
+            let mut original = token.clone();
+            original.position = adjusted_position;
+            original.position_length = parts.len();
+            self.pending.push_back(original);
+        }
+
+        for (part_idx, (start, end)) in parts.iter().copied().enumerate() {
+            let mut part = token.clone();
+            part.offset_from = token.offset_from + start;
+            part.offset_to = token.offset_from + end;
+            part.position = adjusted_position + part_idx;
+            part.position_length = 1;
+            part.text.clear();
+            part.text.push_str(&token.text[start..end]);
+            self.pending.push_back(part);
+        }
+        self.position_offset += parts.len() - 1;
+    }
+}
+
+impl<T: TokenStream> TokenStream for WordDelimiterTokenStream<T> {
+    fn advance(&mut self) -> bool {
+        if !self.refill() {
+            return false;
+        }
+        let Some(token) = self.pending.pop_front() else {
+            debug_assert!(false, "pending token should be available after refill");
+            return false;
+        };
+        self.current = token;
+        true
+    }
+
+    fn token(&self) -> &Token {
+        &self.current
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.current
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{CodeLexTokenizer, TextAnalyzer, Token, WordDelimiterFilter};
+
+    fn collect_tokens(text: &str) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false))
+            .filter(WordDelimiterFilter::new(true, true))
+            .build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_word_delimiter_code_identifiers() {
+        let tokens = collect_tokens("getUserName XMLHttpRequest parseHTML2JSON utf8_reader");
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            texts,
+            vec![
+                "getUserName",
+                "get",
+                "User",
+                "Name",
+                "XMLHttpRequest",
+                "XML",
+                "Http",
+                "Request",
+                "parseHTML2JSON",
+                "parse",
+                "HTML",
+                "2",
+                "JSON",
+                "utf8_reader",
+                "utf",
+                "8",
+                "reader",
+            ]
+        );
+    }
+
+    #[test]
+    fn test_word_delimiter_positions_span_compounds() {
+        let tokens = collect_tokens("getUserName next");
+        let positions = tokens
+            .iter()
+            .map(|token| (token.text.as_str(), token.position, token.position_length))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            positions,
+            vec![
+                ("getUserName", 0, 3),
+                ("get", 0, 1),
+                ("User", 1, 1),
+                ("Name", 2, 1),
+                ("next", 3, 1),
+            ]
+        );
+    }
+}
diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml
index 36ea5facc29..762ca513a19 100644
--- a/rust/lance/Cargo.toml
+++ b/rust/lance/Cargo.toml
@@ -139,6 +139,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"
 
 [features]
 default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"]
+backtrace = ["lance-core/backtrace"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 # Prevent dynamic linking of lzma, which comes from datafusion
 cli = ["dep:clap", "lzma-sys/static"]
@@ -160,6 +161,8 @@ tencent = ["lance-io/tencent"]
 goosefs = ["lance-io/goosefs"]
 tos = ["lance-io/tos"]
 huggingface = ["lance-io/huggingface"]
+# Publish object store metrics via the `metrics` crate.
+metrics = ["lance-io/metrics"]
 geo = ["lance-datafusion/geo", "lance-index/geo"]
 # Enable slow integration tests (disabled by default in CI)
 slow_tests = []
@@ -195,6 +198,10 @@ harness = false
 name = "scan"
 harness = false
 
+[[bench]]
+name = "s3_file_reader_diagnostics"
+harness = false
+
 [[bench]]
 name = "count_pushdown"
 harness = false
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
index 63f9368c3c5..188e2c73ce2 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
@@ -111,11 +111,6 @@ impl Mode {
     fn durable_write(self) -> bool {
         matches!(self, Self::SyncNoIndex | Self::SyncIndexed)
     }
-
-    /// Index update happens inline in `put` (only meaningful when indexed).
-    fn sync_indexed_write(self) -> bool {
-        matches!(self, Self::SyncIndexed)
-    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -428,7 +423,6 @@ fn shard_writer_config(args: &Args, shard_id: Uuid, disable_auto_flush: bool) ->
     };
     let mut config = ShardWriterConfig::new(shard_id)
         .with_durable_write(args.mode.durable_write())
-        .with_sync_indexed_write(args.mode.sync_indexed_write())
         .with_max_memtable_size(args.max_memtable_size)
         .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes)
         .with_max_memtable_rows(max_rows)
@@ -638,7 +632,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result<serde_jso
     let dataset = Dataset::open(uri).await?;
     let shard_id = Uuid::new_v4();
     // Auto-flush disabled so the MemTable holds the full read_rows.
-    // sync_indexed_write + durable_write are both forced on: the read
+    // durable_write is forced on: the read
     // panel measures FTS read latency and consistency, which are
     // properties of the *fully visible* MemTable. The MemTableScanner
     // respects the visibility watermark, and that watermark only advances
@@ -646,9 +640,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result<serde_jso
     // would query a partially-visible MemTable and report garbage. Both
     // are properties of ingestion, not of the read path, so forcing them
     // does not bias the latency/consistency measurement.
-    let config = shard_writer_config(args, shard_id, true)
-        .with_sync_indexed_write(true)
-        .with_durable_write(true);
+    let config = shard_writer_config(args, shard_id, true).with_durable_write(true);
     let writer = dataset.mem_wal_writer(shard_id, config).await?;
 
     let ingest_pool = &corpus[read_seed..];
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
index 08fdb76b95f..462acb5b717 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
@@ -5,11 +5,11 @@
 //!
 //! Sibling of `mem_wal_vector_bench.rs` / `mem_wal_point_lookup_bench.rs`:
 //! same `--phase prepare|search` shape, same `ShardWriter`-based ingestion
-//! of flushed generations + an active memtable, same `--uri` cloud/local
+//! of SSTables + an active memtable, same `--uri` cloud/local
 //! detection, and the same JSON output contract. The payload is real
 //! HuggingFace FineWeb `text` and the query path is
-//! [`LsmFtsSearchPlanner`] (local scoring) over the base table + flushed
-//! generations + active memtable.
+//! [`LsmFtsSearchPlanner`] (local scoring) over the base table + SSTables
+//! + active memtable.
 //!
 //! Each `search` invocation times a query set against the LSM hierarchy
 //! and reports latency percentiles. With `--with-baseline`, it also builds
@@ -108,7 +108,7 @@ struct Args {
     uri: String,
     base_rows: usize,
     max_memtable_rows: usize,
-    flushed_generations: usize,
+    sstables: usize,
     batch_rows: usize,
     queries: usize,
     k: usize,
@@ -126,7 +126,7 @@ impl Default for Args {
             uri: String::new(),
             base_rows: 1_000_000,
             max_memtable_rows: 100_000,
-            flushed_generations: 2,
+            sstables: 2,
             batch_rows: 1_000,
             queries: 200,
             k: 10,
@@ -175,7 +175,7 @@ fn parse_args() -> Result<Args> {
             }
             "--base-rows" => args.base_rows = parse_val(&flag, &value)?,
             "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?,
-            "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?,
+            "--sstables" => args.sstables = parse_val(&flag, &value)?,
             "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?,
             "--queries" => args.queries = parse_val(&flag, &value)?,
             "--k" => args.k = parse_val(&flag, &value)?,
@@ -664,7 +664,7 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
     // covering both the memtable payload and the query-term sample, instead
     // of re-reading the whole base corpus from parquet.
     let active_rows = args.max_memtable_rows / 2;
-    let total_memtable_rows = args.flushed_generations * args.max_memtable_rows + active_rows;
+    let total_memtable_rows = args.sstables * args.max_memtable_rows + active_rows;
     let sample_rows = args.base_rows.min(50_000);
     let load_rows = total_memtable_rows.max(sample_rows);
     println!("loading {load_rows} FineWeb rows for memtable payload + query sample ...");
@@ -688,7 +688,6 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: args.max_memtable_rows * row_bytes * 100,
         max_memtable_rows: args.max_memtable_rows,
         max_memtable_batches: batches_per_gen,
@@ -704,10 +703,8 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         Duration::from_millis(500)
     };
 
-    // Ingest flushed generations + 1 active (50% full).
-    let mut gen_sizes: Vec<usize> = (0..args.flushed_generations)
-        .map(|_| args.max_memtable_rows)
-        .collect();
+    // Ingest SSTables + 1 active (50% full).
+    let mut gen_sizes: Vec<usize> = (0..args.sstables).map(|_| args.max_memtable_rows).collect();
     gen_sizes.push(active_rows);
 
     let id_base = args.base_rows as i64;
@@ -724,19 +721,19 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
             cursor += chunk;
             written += chunk;
         }
-        let is_flushed = gen_idx < args.flushed_generations;
+        let is_sstable = gen_idx < args.sstables;
         println!(
             "  gen {}: wrote {} rows ({})",
             gen_idx + 1,
             gen_rows,
-            if is_flushed { "flushed" } else { "active" }
+            if is_sstable { "sstable" } else { "active" }
         );
-        if is_flushed {
+        if is_sstable {
             tokio::time::sleep(flush_wait).await;
         }
     }
     // Wait for any triggered (sealed) memtable flushes to commit to the
-    // manifest before we snapshot it — otherwise the flushed generations
+    // manifest before we snapshot it — otherwise the SSTables
     // race the read and may not all be visible yet.
     writer.wait_for_flush_drain().await?;
     println!(
@@ -750,17 +747,14 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
     let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
         shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
-        for fg in &m.flushed_generations {
-            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        for sstable in &m.sstables {
+            shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone());
         }
     }
-    let num_flushed = manifest
-        .as_ref()
-        .map(|m| m.flushed_generations.len())
-        .unwrap_or(0);
-    println!("manifest: {num_flushed} flushed generations");
+    let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0);
+    println!("manifest: {num_sstables} SSTables");
 
-    // Flushed generations carry the same maintained secondary indexes as
+    // SSTables carry the same maintained secondary indexes as
     // the active memtable: the flush handler builds them during flush
     // (lance #6901), so each generation already has the FTS index and
     // both scoring modes use the fast indexed path. No manual indexing
@@ -827,7 +821,7 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         "uri_kind": if is_cloud_uri(&args.uri) { "cloud" } else { "local" },
         "base_rows": args.base_rows,
         "max_memtable_rows": args.max_memtable_rows,
-        "flushed_generations": num_flushed,
+        "sstables": num_sstables,
         "active_rows": active_rows,
         "k": args.k,
         "queries": queries.len(),
@@ -848,12 +842,12 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
 
 async fn run(args: Args) -> Result<()> {
     println!(
-        "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} queries={} k={} with_baseline={}",
+        "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} sstables={} queries={} k={} with_baseline={}",
         args.phase.as_str(),
         args.uri,
         args.base_rows,
         args.max_memtable_rows,
-        args.flushed_generations,
+        args.sstables,
         args.queries,
         args.k,
         args.with_baseline,
diff --git a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh
index 7ea692ee0b1..31ededa04e2 100755
--- a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh
+++ b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh
@@ -8,7 +8,7 @@
 #
 # For each (backend, base_rows) the bench's `prepare` phase runs once to
 # write the base dataset + FTS index + MemWAL; then for each k the `search`
-# phase ingests flushed generations + an active memtable through ShardWriter
+# phase ingests SSTables + an active memtable through ShardWriter
 # and times the FTS query panel under both Local and LocalWithGlobalRescore
 # scoring modes.
 #
@@ -23,8 +23,8 @@
 #   CACHE_DIR       FineWeb shard download cache (default <tmpdir>/lance-fineweb-cache)
 #   BASE_ROWS_LIST  space-separated base sizes (default "100000 1000000")
 #   K_LIST          space-separated top-k values (default "10 100")
-#   MAX_MEMTABLE_ROWS  active/flushed memtable cap (default 100000)
-#   GENS_LIST       space-separated flushed-generation counts (default "1 2 5")
+#   MAX_MEMTABLE_ROWS  active/SSTable cap (default 100000)
+#   GENS_LIST       space-separated SSTable counts (default "1 2 5")
 #   QUERIES         queries per config (default 200)
 #   WITH_BASELINE   "1" to also build the merged-index accuracy baseline
 #                   and report local-vs-merged Jaccard (default off)
@@ -117,7 +117,7 @@ for backend in $BACKENDS; do
             --base-rows "$base_rows" --batch-rows 1000 \
             --cache-dir "$CACHE_DIR" || continue
 
-        # search for each (flushed-generations, k)
+        # search for each (SSTables, k)
         for gens in $GENS_LIST; do
             for k in $K_LIST; do
                 name="search_${backend}_${btag}_g${gens}_k${k}"
@@ -136,7 +136,7 @@ for backend in $BACKENDS; do
                     --phase search --uri "$uri" \
                     --base-rows "$base_rows" \
                     --max-memtable-rows "$MAX_MEMTABLE_ROWS" \
-                    --flushed-generations "$gens" \
+                    --sstables "$gens" \
                     --batch-rows 1000 \
                     --queries "$QUERIES" --k "$k" \
                     "${baseline_flag[@]}" \
diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
index 757c2539642..335a41d3bbd 100644
--- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
+++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
@@ -43,8 +43,7 @@ use datafusion::common::ScalarValue;
 use datafusion::prelude::SessionContext;
 use futures::TryStreamExt;
 use lance::dataset::mem_wal::scanner::{
-    FlushedMemTableCache, InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner,
-    ShardSnapshot,
+    InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, ShardSnapshot, SsTableCache,
 };
 use lance::dataset::mem_wal::{DatasetMemWalExt, ShardWriterConfig};
 use lance::dataset::{Dataset, WriteParams};
@@ -422,7 +421,7 @@ enum LanceReadMode {
     Plan,
     /// Probe the active MemTable's BTree index directly and materialize the
     /// row from the BatchStore, bypassing DataFusion. Single-active-memtable
-    /// fast path (no flushed generations); misses fall through as "not found".
+    /// fast path (no SSTables); misses fall through as "not found".
     Fast,
     /// Call the production `LsmPointLookupPlanner::lookup` API, which uses the
     /// direct BTree fast path internally and falls back to the plan path for
@@ -492,27 +491,27 @@ impl KeyType {
 }
 
 /// Where the data under test lives. `Active` = the in-memory active MemTable
-/// (never flushed). `Flushed` = an on-disk flushed generation (a Lance data
+/// (never flushed). `SsTable` = an on-disk SSTable (a Lance data
 /// file + on-disk BTree index, read via the indexed-scan path) vs a single
 /// RocksDB SST on disk.
 #[derive(Debug, Clone, Copy, PartialEq)]
 enum Storage {
     Active,
-    Flushed,
+    SsTable,
 }
 
 impl Storage {
     fn parse(v: &str) -> std::result::Result<Self, String> {
         match v {
             "active" => Ok(Self::Active),
-            "flushed" => Ok(Self::Flushed),
-            _ => Err(format!("unknown storage '{v}', expected active|flushed")),
+            "sstable" => Ok(Self::SsTable),
+            _ => Err(format!("unknown storage '{v}', expected active|sstable")),
         }
     }
     fn as_str(self) -> &'static str {
         match self {
             Self::Active => "active",
-            Self::Flushed => "flushed",
+            Self::SsTable => "sstable",
         }
     }
 }
@@ -553,7 +552,7 @@ struct Args {
     engine: Engine,
     key_type: KeyType,
     storage: Storage,
-    /// Number of flushed generations below the single active MemTable (Lance) /
+    /// Number of SSTables below the single active MemTable (Lance) /
     /// immutable SSTs below the active memtable (RocksDB). 0 = the existing
     /// single-tier behavior. >0 builds a full LSM: rows are split into
     /// `generations+1` parts, the first `generations` are flushed to on-disk
@@ -574,12 +573,12 @@ struct Args {
     /// the caches and hit NVMe. Caps the RocksDB write buffer + uses a small
     /// block cache + compacts to one SST, and drops the OS page cache before
     /// the read phase (both engines). Use with a `--rows`×`--value-size` larger
-    /// than RAM. Only affects the `--storage flushed` path.
+    /// than RAM. Only affects the `--storage sstable` path.
     cold: bool,
-    /// Prewarm all flushed generations (open + warm indexes) into the dataset
+    /// Prewarm all SSTables (open + warm indexes) into the dataset
     /// session before the read phase, via `DatasetMemWalExt::prewarm_mem_wal`.
     /// Default on. `--prewarm false` disables it to measure the lazy-warm
-    /// baseline (the flushed cache is still set, so each generation is opened
+    /// baseline (the SSTable cache is still set, so each generation is opened
     /// on its first gen-key lookup instead of up front). Only affects the Lance
     /// `--storage active` LSM path.
     prewarm: bool,
@@ -792,7 +791,6 @@ async fn run_lance(
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: true,
-        sync_indexed_write: true,
         max_memtable_size: big,
         max_memtable_rows: args.rows * 4 + 1_000_000,
         max_memtable_batches: args.rows / args.batch_rows + 1_000_000,
@@ -834,7 +832,7 @@ async fn run_lance(
                 let n = writer
                     .manifest()
                     .await?
-                    .map(|m| m.flushed_generations.len())
+                    .map(|m| m.sstables.len())
                     .unwrap_or(0);
                 if n > g {
                     break;
@@ -850,10 +848,10 @@ async fn run_lance(
     let n_gens = writer
         .manifest()
         .await?
-        .map(|m| m.flushed_generations.len())
+        .map(|m| m.sstables.len())
         .unwrap_or(0);
     println!(
-        "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, flushed_gens={n_gens}+active)",
+        "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, sstables={n_gens}+active)",
         args.rows, write_s, write_rows_per_s, write_cpu_s
     );
 
@@ -863,8 +861,8 @@ async fn run_lance(
     let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
         shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
-        for fg in &m.flushed_generations {
-            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        for sstable in &m.sstables {
+            shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone());
         }
     }
     // Keep a handle to the active MemTable for the direct fast path before
@@ -872,22 +870,22 @@ async fn run_lance(
     let active = Arc::new(in_memory_refs.active.clone());
     let collector = LsmDataSourceCollector::new(dataset.clone(), vec![shard_snapshot.clone()])
         .with_in_memory_memtables(shard_id, in_memory_refs);
-    // Thread the dataset session + a flushed-dataset cache into the planner, and
-    // prewarm every flushed generation (open + warm its indexes) up front via
+    // Thread the dataset session + an SSTable cache into the planner, and
+    // prewarm every SSTable (open + warm its indexes) up front via
     // the general MemWAL API, so gen-key lookups never re-open a generation per
     // query (the equivalent of RocksDB keeping its DB + SSTs resident). Without
     // this, each plan-path lookup pays a fresh manifest read + Dataset open — a
     // fixed per-lookup cost independent of generation count.
-    let flushed_cache = Arc::new(FlushedMemTableCache::new((gens as u64).max(1)));
+    let sstable_cache = Arc::new(SsTableCache::new((gens as u64).max(1)));
     if args.prewarm {
         dataset
-            .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&flushed_cache))
+            .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&sstable_cache))
             .await?;
     }
     let planner = Arc::new(
         LsmPointLookupPlanner::new(collector, vec![KEY_COL.to_string()], arrow_schema)
             .with_session(dataset.session())
-            .with_flushed_cache(flushed_cache),
+            .with_sstable_cache(sstable_cache),
     );
 
     // Warmup + correctness: a hit key must resolve to exactly one row under
@@ -1206,7 +1204,7 @@ async fn run_lance(
 }
 
 // ----------------------------------------------------------------------
-// Lance flushed (on-disk) engine
+// Lance SSTable (on-disk) engine
 // ----------------------------------------------------------------------
 
 /// Drop the OS page cache so subsequent reads hit storage (cold). Best-effort:
@@ -1221,7 +1219,7 @@ fn drop_page_cache() {
 /// One indexed point lookup via the **DataFusion** path: `scan().filter("id =
 /// key")` parses + plans + executes a query per lookup (uses the on-disk BTree
 /// index). Returns the matched row count.
-async fn flushed_probe(dataset: &Dataset, key: i64) -> Result<usize> {
+async fn sstable_probe(dataset: &Dataset, key: i64) -> Result<usize> {
     use futures::StreamExt;
     let mut scanner = dataset.scan();
     scanner.filter(&format!("{KEY_COL} = {key}"))?;
@@ -1235,8 +1233,8 @@ async fn flushed_probe(dataset: &Dataset, key: i64) -> Result<usize> {
 
 /// One point lookup via the **direct** path: search the on-disk BTree scalar
 /// index for the row id, then `take` that row — bypassing DataFusion plan
-/// construction. Diagnostic for how much of the flushed read cost is the plan.
-async fn flushed_probe_direct(
+/// construction. Diagnostic for how much of the SSTable read cost is the plan.
+async fn sstable_probe_direct(
     dataset: &Dataset,
     scalar_index: &Arc<dyn lance_index::scalar::ScalarIndex>,
     key: i64,
@@ -1257,25 +1255,25 @@ async fn flushed_probe_direct(
     Ok(batch.num_rows())
 }
 
-/// Dispatch a flushed point lookup: `direct` = on-disk BTree index search + take
+/// Dispatch an SSTable point lookup: `direct` = on-disk BTree index search + take
 /// (no DataFusion); otherwise the DataFusion `scan().filter()` path.
-async fn flushed_lookup(
+async fn sstable_lookup(
     dataset: &Dataset,
     scalar_index: &Arc<dyn lance_index::scalar::ScalarIndex>,
     key: i64,
     direct: bool,
 ) -> Result<usize> {
     if direct {
-        flushed_probe_direct(dataset, scalar_index, key).await
+        sstable_probe_direct(dataset, scalar_index, key).await
     } else {
-        flushed_probe(dataset, key).await
+        sstable_probe(dataset, key).await
     }
 }
 
-/// Batched flushed lookup over a chunk of keys: `direct` searches the index for
+/// Batched SSTable lookup over a chunk of keys: `direct` searches the index for
 /// each key then issues one `take_rows` for all; otherwise one DataFusion scan
 /// with `id IN (...)`. Returns the total matched row count.
-async fn flushed_batch(
+async fn sstable_batch(
     dataset: &Dataset,
     scalar_index: &Arc<dyn lance_index::scalar::ScalarIndex>,
     keys: &[i64],
@@ -1321,11 +1319,11 @@ async fn flushed_batch(
     }
 }
 
-/// Flushed Lance: write all rows as one on-disk Lance dataset with a BTree
+/// SSTable Lance: write all rows as one on-disk Lance dataset with a BTree
 /// scalar index — the exact artifact a MemTable flush emits (forward-written
 /// data file + on-disk BTree index) — then point-lookup through the indexed
 /// scan path. Int keys only (the SQL filter literal is the integer).
-async fn run_lance_flushed(
+async fn run_lance_sstable(
     args: &Args,
     insert_order: &[i64],
     queries: &[(i64, bool)],
@@ -1333,12 +1331,12 @@ async fn run_lance_flushed(
     assert_eq!(
         args.key_type,
         KeyType::Int,
-        "flushed mode currently supports --key-type int only"
+        "sstable mode currently supports --key-type int only"
     );
     let sampler = RssSampler::start();
     let key_type = args.key_type;
     let schema = make_schema(key_type);
-    let uri = format!("{}/lance_flushed", args.uri.trim_end_matches('/'));
+    let uri = format!("{}/lance_sstable", args.uri.trim_end_matches('/'));
     let _ = std::fs::remove_dir_all(&uri);
 
     // --- write + flush: build the on-disk data file + BTree index ---
@@ -1379,13 +1377,13 @@ async fn run_lance_flushed(
             .iter()
             .find(|i| i.name == BTREE_INDEX_NAME)
             .map(|i| i.uuid.to_string())
-            .ok_or_else(|| lance_core::Error::internal("flushed: btree index not found"))?;
+            .ok_or_else(|| lance_core::Error::internal("sstable: btree index not found"))?;
         dataset
             .open_scalar_index(KEY_COL, &uuid, &NoOpMetricsCollector)
             .await?
     };
     println!(
-        "[lance] flushed read path = {}",
+        "[lance] sstable read path = {}",
         if direct {
             "direct btree-index search + take"
         } else {
@@ -1395,8 +1393,8 @@ async fn run_lance_flushed(
 
     // warmup + correctness: a hit resolves to exactly one row via the index.
     if let Some((probe, _)) = queries.iter().find(|(_, h)| *h) {
-        let n = flushed_lookup(&dataset, &scalar_index, *probe, direct).await?;
-        assert_eq!(n, 1, "flushed warmup lookup for key {probe} returned {n}");
+        let n = sstable_lookup(&dataset, &scalar_index, *probe, direct).await?;
+        assert_eq!(n, 1, "sstable warmup lookup for key {probe} returned {n}");
     }
 
     // Cold mode: drop the OS page cache so reads hit NVMe (data > RAM assumed).
@@ -1405,7 +1403,7 @@ async fn run_lance_flushed(
         println!("[lance] dropped page cache (cold reads from NVMe)");
     }
 
-    // --- batch-get path: gather `batch_get` keys per call from the flushed gen ---
+    // --- batch-get path: gather `batch_get` keys per call from the SSTable ---
     if args.batch_get > 0 {
         let bg = args.batch_get;
         let hit_keys: Vec<i64> = queries
@@ -1419,7 +1417,7 @@ async fn run_lance_flushed(
         let t = Instant::now();
         for chunk in hit_keys.chunks(bg) {
             let t0 = Instant::now();
-            found_total += flushed_batch(&dataset, &scalar_index, chunk, direct).await?;
+            found_total += sstable_batch(&dataset, &scalar_index, chunk, direct).await?;
             latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0);
         }
         let read_qps_1t = hit_keys.len() as f64 / t.elapsed().as_secs_f64().max(1e-9);
@@ -1439,7 +1437,7 @@ async fn run_lance_flushed(
                     let chunks: Vec<&[i64]> = keys.chunks(bg).collect();
                     let mut i = shard;
                     while i < chunks.len() {
-                        let _ = flushed_batch(&dataset, &si, chunks[i], direct).await;
+                        let _ = sstable_batch(&dataset, &si, chunks[i], direct).await;
                         i += threads;
                     }
                 }));
@@ -1456,7 +1454,7 @@ async fn run_lance_flushed(
             args.threads, stats.p50_us, stats.p99_us
         );
         return Ok(EngineResult {
-            engine: "lance-flushed-batch",
+            engine: "lance-sstable-batch",
             write_rows_per_s,
             write_cpu_s,
             read_p50_us: stats.p50_us,
@@ -1481,7 +1479,7 @@ async fn run_lance_flushed(
     let t_read = Instant::now();
     for &(key, expect_hit) in queries {
         let t0 = Instant::now();
-        let n = flushed_lookup(&dataset, &scalar_index, key, direct).await?;
+        let n = sstable_lookup(&dataset, &scalar_index, key, direct).await?;
         latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0);
         if expect_hit {
             assert_eq!(n, 1, "expected hit for key {key}, got {n}");
@@ -1511,7 +1509,7 @@ async fn run_lance_flushed(
             handles.push(tokio::spawn(async move {
                 let mut i = shard;
                 while i < keys.len() {
-                    let _ = flushed_lookup(&dataset, &scalar_index, keys[i], direct).await;
+                    let _ = sstable_lookup(&dataset, &scalar_index, keys[i], direct).await;
                     i += threads;
                 }
             }));
@@ -1536,7 +1534,7 @@ async fn run_lance_flushed(
     );
 
     Ok(EngineResult {
-        engine: "lance-flushed",
+        engine: "lance-sstable",
         write_rows_per_s,
         write_cpu_s,
         read_p50_us: stats.p50_us,
@@ -1585,7 +1583,7 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re
     opts.set_min_write_buffer_number_to_merge(2);
     opts.set_disable_auto_compactions(true);
     opts.set_db_write_buffer_size(write_buf);
-    // Block cache for the `--storage flushed` SST reads. Warm: large enough to
+    // Block cache for the `--storage sstable` SST reads. Warm: large enough to
     // hold the SST index/filter + hot data blocks. Cold: small (128MB) so data
     // blocks miss the cache and reads go to NVMe (index/filter stay in memory).
     {
@@ -1654,17 +1652,17 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re
         write_buf >> 20
     );
 
-    // Single-tier `--storage flushed` (no extra generations): flush the active
+    // Single-tier `--storage sstable` (no extra generations): flush the active
     // to one SST (compact to one if cold). With generations>0 the per-chunk
     // flushes already produced N separate L0 SSTs + the active memtable.
-    if gens == 0 && args.storage == Storage::Flushed {
+    if gens == 0 && args.storage == Storage::SsTable {
         db.flush()
             .map_err(|e| lance_core::Error::io(format!("rocksdb flush: {e}")))?;
         if args.cold {
             db.compact_range::<&[u8], &[u8]>(None, None);
         }
     }
-    if args.storage == Storage::Flushed || gens > 0 {
+    if args.storage == Storage::SsTable || gens > 0 {
         let n_sst = db
             .property_int_value("rocksdb.num-files-at-level0")
             .ok()
@@ -1939,7 +1937,7 @@ async fn run(args: Args) -> Result<()> {
     if matches!(args.engine, Engine::Lance | Engine::Both) {
         let res = match args.storage {
             Storage::Active => run_lance(&args, &insert_order, &queries).await?,
-            Storage::Flushed => run_lance_flushed(&args, &insert_order, &queries).await?,
+            Storage::SsTable => run_lance_sstable(&args, &insert_order, &queries).await?,
         };
         results.push(res);
     }
diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
index cb9e6413ac1..0b92d1d9c73 100644
--- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
+++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
@@ -4,8 +4,8 @@
 //! Standalone CLI benchmark for PK-based point lookups across LSM levels.
 //!
 //! Measures lookup latency against three tiers of the LSM tree:
-//!   - Base table (on-disk, merged data)
-//!   - Flushed MemTable generations (on-disk L0)
+//!   - Base table (on-disk, compacted data)
+//!   - SSTables (on-disk L0)
 //!   - Active MemTable (in-memory write buffer)
 //!
 //! Two phases, selected with `--phase`:
@@ -151,7 +151,7 @@ struct Args {
     uri: String,
     base_rows: usize,
     max_memtable_rows: usize,
-    flushed_generations: usize,
+    sstables: usize,
     batch_rows: usize,
     queries: usize,
     output: Option<PathBuf>,
@@ -164,7 +164,7 @@ impl Default for Args {
             uri: String::new(),
             base_rows: 1_000_000,
             max_memtable_rows: 100_000,
-            flushed_generations: 2,
+            sstables: 2,
             batch_rows: 1_000,
             queries: 500,
             output: None,
@@ -205,7 +205,7 @@ fn parse_args() -> Result<Args> {
             }
             "--base-rows" => args.base_rows = parse_val(&flag, &value)?,
             "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?,
-            "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?,
+            "--sstables" => args.sstables = parse_val(&flag, &value)?,
             "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?,
             "--queries" => args.queries = parse_val(&flag, &value)?,
             "--output" => args.output = Some(PathBuf::from(value)),
@@ -276,11 +276,11 @@ fn is_cloud_uri(uri: &str) -> bool {
 fn generate_lookup_ids(
     base_rows: usize,
     max_memtable_rows: usize,
-    flushed_generations: usize,
+    sstables: usize,
     queries: usize,
 ) -> (Vec<Vec<i64>>, Vec<&'static str>) {
-    let flushed_total = flushed_generations * max_memtable_rows;
-    let active_start = base_rows + flushed_total;
+    let sstable_total = sstables * max_memtable_rows;
+    let active_start = base_rows + sstable_total;
     let active_end = active_start + max_memtable_rows / 2;
 
     let mut groups = Vec::new();
@@ -296,19 +296,19 @@ fn generate_lookup_ids(
     groups.push(base_ids);
     names.push("base");
 
-    // Flushed IDs (only if there are flushed generations)
-    if flushed_generations > 0 {
-        let flushed_start = base_rows;
-        let flushed_end = base_rows + flushed_total;
-        let flushed_ids: Vec<i64> = (0..queries)
+    // SSTable IDs (only if there are SSTables)
+    if sstables > 0 {
+        let sstable_start = base_rows;
+        let sstable_end = base_rows + sstable_total;
+        let sstable_ids: Vec<i64> = (0..queries)
             .map(|i| {
-                let range = flushed_end - flushed_start;
+                let range = sstable_end - sstable_start;
                 let step = range.max(1) / queries.max(1);
-                (flushed_start + (i * step) % range) as i64
+                (sstable_start + (i * step) % range) as i64
             })
             .collect();
-        groups.push(flushed_ids);
-        names.push("flushed");
+        groups.push(sstable_ids);
+        names.push("sstable");
     }
 
     // Active memtable IDs
@@ -338,7 +338,6 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: max_memtable_rows * 200,
         max_memtable_rows,
         max_wal_flush_interval: Some(Duration::from_secs(60)),
@@ -354,11 +353,11 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
     };
 
     let id_base = args.base_rows as i64;
-    let num_flushed = args.flushed_generations;
+    let num_sstables = args.sstables;
     let active_rows = max_memtable_rows / 2;
 
-    // Ingest flushed generations (each triggers a flush) + 1 active (50% full)
-    let mut gen_sizes: Vec<usize> = (0..num_flushed).map(|_| max_memtable_rows).collect();
+    // Ingest SSTables (each triggers a flush) + 1 active (50% full)
+    let mut gen_sizes: Vec<usize> = (0..num_sstables).map(|_| max_memtable_rows).collect();
     gen_sizes.push(active_rows);
 
     let mut cursor = 0usize;
@@ -371,22 +370,22 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
             writer.put(vec![batch]).await?;
             cursor += rows;
         }
-        let is_flushed = gen_idx < num_flushed;
+        let is_sstable = gen_idx < num_sstables;
         println!(
             "  gen {}: wrote {} rows ({}) cursor={}",
             gen_idx + 1,
             gen_rows,
-            if is_flushed { "flushed" } else { "active" },
+            if is_sstable { "sstable" } else { "active" },
             cursor,
         );
-        if is_flushed {
+        if is_sstable {
             tokio::time::sleep(flush_wait).await;
         }
     }
 
     println!(
-        "ingested {} rows total ({} flushed gens + active)",
-        cursor, num_flushed
+        "ingested {} rows total ({} SSTables + active)",
+        cursor, num_sstables
     );
 
     let manifest = writer.manifest().await.unwrap();
@@ -396,18 +395,15 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
     let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
         shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
-        for fg in &m.flushed_generations {
-            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        for sstable in &m.sstables {
+            shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone());
         }
     }
 
-    let num_flushed = manifest
-        .as_ref()
-        .map(|m| m.flushed_generations.len())
-        .unwrap_or(0);
+    let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0);
     println!(
-        "manifest: {} flushed generations, current_generation={}",
-        num_flushed,
+        "manifest: {} SSTables, current_generation={}",
+        num_sstables,
         manifest.as_ref().map(|m| m.current_generation).unwrap_or(0)
     );
 
@@ -418,8 +414,12 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
     let planner = LsmPointLookupPlanner::new(collector, pk_columns, arrow_schema);
 
     // Generate lookup IDs for each category
-    let (id_groups, category_names) =
-        generate_lookup_ids(args.base_rows, max_memtable_rows, num_flushed, args.queries);
+    let (id_groups, category_names) = generate_lookup_ids(
+        args.base_rows,
+        max_memtable_rows,
+        num_sstables,
+        args.queries,
+    );
 
     let session_ctx = SessionContext::new();
     let task_ctx = session_ctx.task_ctx();
@@ -476,7 +476,7 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
     output.insert("phase".into(), json!("lookup"));
     output.insert("base_rows".into(), json!(args.base_rows));
     output.insert("max_memtable_rows".into(), json!(max_memtable_rows));
-    output.insert("flushed_generations".into(), json!(num_flushed));
+    output.insert("sstables".into(), json!(num_sstables));
     output.insert("active_rows".into(), json!(active_rows));
     output.insert("queries_per_category".into(), json!(args.queries));
     for (key, val) in &results {
@@ -491,12 +491,12 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
 
 async fn run(args: Args) -> Result<()> {
     println!(
-        "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} batch_rows={} queries={}",
+        "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} sstables={} batch_rows={} queries={}",
         args.phase.as_str(),
         args.uri,
         args.base_rows,
         args.max_memtable_rows,
-        args.flushed_generations,
+        args.sstables,
         args.batch_rows,
         args.queries,
     );
diff --git a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py
index e3b81123ece..f93a06e5188 100644
--- a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py
+++ b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
-"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the flushed-memtable index)
+"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the SSTable index)
 vs DiskANN vs FAISS, all backed by local NVMe.
 
 faiss/diskannpy/lance bundle conflicting tcmalloc/MKL/OpenMP and crash if
@@ -18,7 +18,12 @@
 recall@10 vs p50/p99 latency and QPS. The Lance index is served fully cached
 (large index_cache_size_bytes).
 """
-import argparse, json, os, time
+
+import argparse
+import json
+import os
+import time
+
 import numpy as np
 
 K = 10
@@ -27,7 +32,9 @@
 DIM = 1536
 EF_SWEEP = [16, 32, 64, 128, 256]
 HF_TREE = "https://huggingface.co/api/datasets/KShivendu/dbpedia-entities-openai-1M/tree/main/data"
-HF_BASE = "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/"
+HF_BASE = (
+    "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/"
+)
 
 
 def data_dir(base, rows):
@@ -42,10 +49,13 @@ def normalize(x):
 
 # ---------------- prepare ----------------
 def load_corpus(cache_dir, needed):
-    import requests, pyarrow.parquet as pq
+    import pyarrow.parquet as pq
+    import requests
+
     os.makedirs(cache_dir, exist_ok=True)
     shards = sorted(
-        e["path"] for e in requests.get(HF_TREE, timeout=60).json()
+        e["path"]
+        for e in requests.get(HF_TREE, timeout=60).json()
         if e["type"] == "file" and e["path"].endswith(".parquet")
     )
     out = np.empty((needed, DIM), dtype=np.float32)
@@ -62,7 +72,7 @@ def load_corpus(cache_dir, needed):
         col = pq.read_table(local, columns=["openai"]).column("openai")
         arr = np.stack(col.to_pylist()).astype(np.float32)
         take = min(len(arr), needed - n)
-        out[n:n + take] = arr[:take]
+        out[n : n + take] = arr[:take]
         n += take
         print(f"  shard {os.path.basename(rel)} -> {take} (cum {n})", flush=True)
     assert n == needed, f"only got {n}/{needed}"
@@ -73,7 +83,6 @@ def numpy_ground_truth(corpus, queries):
     gt = np.empty((len(queries), K), dtype=np.int64)
     # corpus is normalized -> cosine == inner product; chunk over corpus.
     chunk = 200_000
-    sims_top = None
     # Compute full similarity in query-major chunks to bound memory.
     sim = np.zeros((len(queries), len(corpus)), dtype=np.float32)
     for s in range(0, len(corpus), chunk):
@@ -98,10 +107,13 @@ def cmd_prepare(args):
     rng = np.random.default_rng(SEED)
     qidx = rng.choice(args.rows, size=NUM_QUERIES, replace=False)
     queries = corpus[qidx].copy()
-    print(f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...", flush=True)
+    print(
+        f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...",
+        flush=True,
+    )
     t = time.perf_counter()
     gt = numpy_ground_truth(corpus, queries)
-    print(f"  GT in {time.perf_counter()-t:.1f}s", flush=True)
+    print(f"  GT in {time.perf_counter() - t:.1f}s", flush=True)
     np.save(os.path.join(d, "corpus.npy"), corpus)
     np.save(os.path.join(d, "queries.npy"), queries)
     np.save(os.path.join(d, "gt.npy"), gt)
@@ -110,7 +122,9 @@ def cmd_prepare(args):
 
 # ---------------- shared run helpers ----------------
 def recall_at_k(gt, got):
-    return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / (len(gt) * K)
+    return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / (
+        len(gt) * K
+    )
 
 
 def latency_qps(query_fn, queries, repeats=3):
@@ -133,56 +147,97 @@ def sweep(name, make_q, params, queries, gt):
         got = np.stack([qf(v) for v in queries])
         rec = recall_at_k(gt, got)
         p50, p99, qps = latency_qps(qf, queries)
-        rows.append({"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps})
-        print(f"  {name} param={p} recall={rec:.4f} p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}", flush=True)
+        rows.append(
+            {"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps}
+        )
+        print(
+            f"  {name} param={p} recall={rec:.4f} "
+            f"p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}",
+            flush=True,
+        )
     return rows
 
 
 # ---------------- systems ----------------
 def run_lance(base, rows, corpus, queries, gt):
-    import lance, pyarrow as pa, shutil
+    import shutil
+
+    import lance
+    import pyarrow as pa
+
     uri = os.path.join(base, f"lance_{rows}")
     shutil.rmtree(uri, ignore_errors=True)
-    vecs = pa.FixedSizeListArray.from_arrays(pa.array(corpus.reshape(-1), type=pa.float32()), DIM)
+    vecs = pa.FixedSizeListArray.from_arrays(
+        pa.array(corpus.reshape(-1), type=pa.float32()), DIM
+    )
     tbl = pa.table({"id": pa.array(np.arange(rows, dtype=np.int64)), "vec": vecs})
     ds = lance.write_dataset(tbl, uri, mode="overwrite")
-    # The flushed memtable index is a SINGLE-partition HNSW+SQ, so model it with
+    # The SSTable index is a SINGLE-partition HNSW+SQ, so model it with
     # num_partitions=1 (nprobes=1); ef is the search knob, like DiskANN/FAISS.
     t = time.perf_counter()
-    ds.create_index("vec", "IVF_HNSW_SQ", metric="cosine", num_partitions=1,
-                    m=20, ef_construction=150)
+    ds.create_index(
+        "vec",
+        "IVF_HNSW_SQ",
+        metric="cosine",
+        num_partitions=1,
+        m=20,
+        ef_construction=150,
+    )
     build_s = time.perf_counter() - t
     ds = lance.dataset(uri, index_cache_size_bytes=48 * 1024**3)
 
     def make_q(ef):
         def q(v):
-            return ds.to_table(nearest={"column": "vec", "q": v, "k": K,
-                                        "nprobes": 1, "ef": ef},
-                               columns=["id"]).column("id").to_numpy()
+            return (
+                ds.to_table(
+                    nearest={"column": "vec", "q": v, "k": K, "nprobes": 1, "ef": ef},
+                    columns=["id"],
+                )
+                .column("id")
+                .to_numpy()
+            )
+
         return q
-    return {"build_s": build_s, "nlist": 1, "sweep": sweep("lance", make_q, None, queries, gt)}
+
+    return {
+        "build_s": build_s,
+        "nlist": 1,
+        "sweep": sweep("lance", make_q, None, queries, gt),
+    }
 
 
-def run_lance_flushed(base, rows, corpus, queries, gt, lance_path, id_offset, column):
-    # Open a flushed MemTable generation directly from its dataset path and
+def run_lance_sstable(base, rows, corpus, queries, gt, lance_path, id_offset, column):
+    # Open an SSTable generation directly from its dataset path and
     # benchmark its on-disk IVF_HNSW_SQ index (single partition), fully cached.
     import lance
+
     ds = lance.dataset(lance_path, index_cache_size_bytes=48 * 1024**3)
 
     def make_q(ef):
         def q(v):
-            ids = ds.to_table(nearest={"column": column, "q": v, "k": K,
-                                       "nprobes": 1, "ef": ef},
-                              columns=["id"]).column("id").to_numpy()
-            return ids - id_offset  # map flushed-gen id -> corpus index
+            ids = (
+                ds.to_table(
+                    nearest={"column": column, "q": v, "k": K, "nprobes": 1, "ef": ef},
+                    columns=["id"],
+                )
+                .column("id")
+                .to_numpy()
+            )
+            return ids - id_offset  # map SSTable id -> corpus index
+
         return q
-    return {"lance_path": lance_path, "id_offset": id_offset,
-            "sweep": sweep("lance_flushed", make_q, None, queries, gt)}
+
+    return {
+        "lance_path": lance_path,
+        "id_offset": id_offset,
+        "sweep": sweep("lance_sstable", make_q, None, queries, gt),
+    }
 
 
 def run_faiss(base, rows, corpus, queries, gt):
     # Full-precision HNSW reference (shows what no quantization buys).
     import faiss
+
     index = faiss.IndexHNSWFlat(DIM, 32, faiss.METRIC_INNER_PRODUCT)
     index.hnsw.efConstruction = 200
     t = time.perf_counter()
@@ -194,16 +249,20 @@ def make_q(ef):
         def q(v):
             index.hnsw.efSearch = ef
             return index.search(v.reshape(1, -1), K)[1][0]
+
         return q
+
     return {"build_s": build_s, "sweep": sweep("faiss", make_q, None, queries, gt)}
 
 
 def run_faiss_sq(base, rows, corpus, queries, gt):
     # HNSW + 8-bit scalar quantization — apples-to-apples with Lance IVF_HNSW_SQ.
     import faiss
+
     try:
-        index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32,
-                                  faiss.METRIC_INNER_PRODUCT)
+        index = faiss.IndexHNSWSQ(
+            DIM, faiss.ScalarQuantizer.QT_8bit, 32, faiss.METRIC_INNER_PRODUCT
+        )
     except Exception:
         # Fall back to L2; on unit-normalized vectors L2 ranking == cosine.
         index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32)
@@ -218,28 +277,44 @@ def make_q(ef):
         def q(v):
             index.hnsw.efSearch = ef
             return index.search(v.reshape(1, -1), K)[1][0]
+
         return q
+
     return {"build_s": build_s, "sweep": sweep("faiss_sq", make_q, None, queries, gt)}
 
 
 def run_diskann(base, rows, corpus, queries, gt):
     import diskannpy as dap
+
     idx_dir = os.path.join(base, f"diskann_{rows}")
     os.makedirs(idx_dir, exist_ok=True)
     t = time.perf_counter()
     dap.build_memory_index(
-        data=corpus, distance_metric="cosine", index_directory=idx_dir,
-        index_prefix="ann", complexity=150, graph_degree=64,
-        num_threads=0, alpha=1.2, use_pq_build=False, num_pq_bytes=0,
+        data=corpus,
+        distance_metric="cosine",
+        index_directory=idx_dir,
+        index_prefix="ann",
+        complexity=150,
+        graph_degree=64,
+        num_threads=0,
+        alpha=1.2,
+        use_pq_build=False,
+        num_pq_bytes=0,
     )
     build_s = time.perf_counter() - t
-    idx = dap.StaticMemoryIndex(index_directory=idx_dir, index_prefix="ann",
-                                num_threads=0, initial_search_complexity=256)
+    idx = dap.StaticMemoryIndex(
+        index_directory=idx_dir,
+        index_prefix="ann",
+        num_threads=0,
+        initial_search_complexity=256,
+    )
 
     def make_q(L):
         def q(v):
             return idx.search(v, k_neighbors=K, complexity=max(L, K)).identifiers
+
         return q
+
     return {"build_s": build_s, "sweep": sweep("diskann", make_q, None, queries, gt)}
 
 
@@ -249,12 +324,24 @@ def cmd_run(args):
     queries = np.load(os.path.join(d, "queries.npy"))
     gt = np.load(os.path.join(d, "gt.npy"))
     print(f"=== {args.system} rows={args.rows} corpus={len(corpus)} ===", flush=True)
-    if args.system == "lance_flushed":
-        res = run_lance_flushed(args.base, args.rows, corpus, queries, gt,
-                                args.lance_path, args.id_offset, args.column)
+    if args.system == "lance_sstable":
+        res = run_lance_sstable(
+            args.base,
+            args.rows,
+            corpus,
+            queries,
+            gt,
+            args.lance_path,
+            args.id_offset,
+            args.column,
+        )
     else:
-        fn = {"lance": run_lance, "faiss": run_faiss, "faiss_sq": run_faiss_sq,
-              "diskann": run_diskann}[args.system]
+        fn = {
+            "lance": run_lance,
+            "faiss": run_faiss,
+            "faiss_sq": run_faiss_sq,
+            "diskann": run_diskann,
+        }[args.system]
         res = fn(args.base, args.rows, corpus, queries, gt)
     res["rows"] = args.rows
     res["system"] = args.system
@@ -267,9 +354,16 @@ def cmd_run(args):
 def main():
     ap = argparse.ArgumentParser()
     sub = ap.add_subparsers(dest="cmd", required=True)
-    p = sub.add_parser("prepare"); p.add_argument("--rows", type=int, required=True); p.add_argument("--base", required=True)
-    r = sub.add_parser("run"); r.add_argument("--rows", type=int, required=True); r.add_argument("--base", required=True); r.add_argument("--system", required=True)
-    r.add_argument("--lance-path", default=None); r.add_argument("--id-offset", type=int, default=0); r.add_argument("--column", default="vector")
+    p = sub.add_parser("prepare")
+    p.add_argument("--rows", type=int, required=True)
+    p.add_argument("--base", required=True)
+    r = sub.add_parser("run")
+    r.add_argument("--rows", type=int, required=True)
+    r.add_argument("--base", required=True)
+    r.add_argument("--system", required=True)
+    r.add_argument("--lance-path", default=None)
+    r.add_argument("--id-offset", type=int, default=0)
+    r.add_argument("--column", default="vector")
     args = ap.parse_args()
     (cmd_prepare if args.cmd == "prepare" else cmd_run)(args)
 
diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
index 7b9f35b73c9..625900cf834 100644
--- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
+++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
@@ -355,7 +355,7 @@ async fn run_checkpoint(
     let temp = tempfile::tempdir().map_err(|e| lance_core::Error::io(format!("tempdir: {}", e)))?;
     // When BENCH_URI_BASE is set, write to a persistent path and flush the
     // MemTable to an on-disk generation (instead of querying the active
-    // MemTable), printing the flushed generation's dataset path for a
+    // MemTable), printing the SSTable's dataset path for a
     // downstream direct-read benchmark.
     let flush_base = std::env::var("BENCH_URI_BASE").ok();
     let local_dir = flush_base.as_ref().map(|b| format!("{}/cp_{}", b, cp));
@@ -389,7 +389,6 @@ async fn run_checkpoint(
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: true,
         max_memtable_size: cp.saturating_mul(row_size_estimate).saturating_mul(4),
         max_memtable_rows: cp.saturating_mul(2),
         max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000),
@@ -416,11 +415,11 @@ async fn run_checkpoint(
         writer.put(vec![batch]).await?;
     }
 
-    let target_batch_pos = total_batches.saturating_sub(1);
+    let target_indexed_count = total_batches;
     let mut spins = 0u64;
     loop {
         let active = writer.active_memtable_ref().await?;
-        if active.index_store.max_visible_batch_position() >= target_batch_pos {
+        if active.index_store.indexed_count() >= target_indexed_count {
             break;
         }
         drop(active);
@@ -450,9 +449,14 @@ async fn run_checkpoint(
         let mut waited = 0u64;
         let gen_path = loop {
             if let Some(m) = writer.manifest().await?
-                && let Some(fg) = m.flushed_generations.last()
+                && let Some(sstable) = m.sstables.last()
             {
-                break format!("{}/_mem_wal/{}/{}", dir, shard_id.as_hyphenated(), fg.path);
+                break format!(
+                    "{}/_mem_wal/{}/{}",
+                    dir,
+                    shard_id.as_hyphenated(),
+                    sstable.path
+                );
             }
             tokio::time::sleep(Duration::from_millis(100)).await;
             waited += 1;
@@ -467,7 +471,7 @@ async fn run_checkpoint(
             }
         };
         println!(
-            "FLUSHED_OK cp={} id_offset={} flush_s={:.2} path={}",
+            "SSTABLE_OK cp={} id_offset={} flush_s={:.2} path={}",
             cp,
             id_offset,
             seal_start.elapsed().as_secs_f64(),
@@ -476,7 +480,7 @@ async fn run_checkpoint(
         std::io::stdout().flush().ok();
         writer.close().await?;
 
-        // Item 4: open the flushed generation directly from its dataset path and
+        // Item 4: open the SSTable directly from its dataset path and
         // benchmark its on-disk IVF_HNSW_SQ read in Rust (single-thread per-query
         // latency + recall vs brute force), sweeping ef. nprobes=1 (single part).
         let gen_uri = format!("file://{}", gen_path);
@@ -526,7 +530,7 @@ async fn run_checkpoint(
             let p99_q = lat[lat.len() * 99 / 100];
             let mean_recall = recall_sum / num_queries as f64;
             println!(
-                "FLUSHED_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}",
+                "SSTABLE_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}",
                 cp, ef, mean_recall, median_q, p99_q
             );
             std::io::stdout().flush().ok();
@@ -537,13 +541,13 @@ async fn run_checkpoint(
 
         // Raw-index path: call VectorIndex::search() directly (partition-find +
         // HNSW+SQ search, returns _rowid/_distance) bypassing the DataFusion
-        // scanner/take/projection. The RAW_READ vs FLUSHED_READ latency delta at
+        // scanner/take/projection. The RAW_READ vs SSTABLE_READ latency delta at
         // matched ef isolates the DataFusion per-query overhead from the actual
         // index search cost.
         let idx_metas = gen_ds.load_indices_by_name(VECTOR_INDEX_NAME).await?;
         let uuid = idx_metas
             .first()
-            .ok_or_else(|| lance_core::Error::io("flushed gen has no vector index".to_string()))?
+            .ok_or_else(|| lance_core::Error::io("SSTable has no vector index".to_string()))?
             .uuid;
         let vidx = gen_ds
             .open_vector_index(VECTOR_COL, &uuid, &NoOpMetricsCollector)
@@ -595,7 +599,7 @@ async fn run_checkpoint(
                 };
                 // IVFIndex::search is intentionally unimplemented (top-level does
                 // partition-aware search); replicate the ANN exec node: pick the
-                // closest partition then search it. Single-partition flushed gen.
+                // closest partition then search it. Single-partition SSTable.
                 let t = Instant::now();
                 let (parts, _) = vidx.find_partitions(&query)?;
                 let pid = parts.value(0) as usize;
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index 8bff64dfcca..c6fb42834a3 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -10,7 +10,9 @@
 //!    index maintained, opens `mem_wal_writer` with a `ShardWriterConfig`
 //!    sized to hold the largest checkpoint without flushing, and times
 //!    `writer.put(batch)` calls. Index updates therefore go through the
-//!    parallel `IndexStore::insert_batches_parallel` path. At each
+//!    `IndexStore::insert_batches` path, which indexes inline at or below
+//!    `PARALLEL_INDEX_MIN_ROWS` rows and spawns a thread per index above it —
+//!    so the checkpoint sizes here straddle that crossover. At each
 //!    checkpoint, queries are issued against `active_memtable_ref()` via
 //!    `MemTableScanner::nearest`.
 //!
@@ -212,7 +214,6 @@ async fn main() -> lance_core::Result<()> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write,
-        sync_indexed_write: true,
         max_memtable_size: max_rows.saturating_mul(row_size_estimate).saturating_mul(4),
         max_memtable_rows: max_rows.saturating_mul(2),
         max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000),
@@ -253,7 +254,7 @@ async fn main() -> lance_core::Result<()> {
 
         while next_cp_idx < checkpoints.len() && total_inserted >= checkpoints[next_cp_idx] {
             let cp = checkpoints[next_cp_idx];
-            let target_batch_pos = (cp / batch_size).saturating_sub(1);
+            let target_indexed_count = cp / batch_size;
             // The WAL flush handler only updates the index watermark when a
             // flush is triggered, and the time-based trigger inside the
             // writer runs only when `put()` is called. After the final put
@@ -265,7 +266,7 @@ async fn main() -> lance_core::Result<()> {
             let mut spins = 0u64;
             loop {
                 let active = writer.active_memtable_ref().await?;
-                if active.index_store.max_visible_batch_position() >= target_batch_pos {
+                if active.index_store.indexed_count() >= target_indexed_count {
                     break;
                 }
                 drop(active);
@@ -375,12 +376,11 @@ async fn measure_flush(
     memtable.set_indexes(registry);
 
     let total_batches = cp.div_ceil(batch_size);
-    for (wal_pos, i) in (0_u64..).zip(0..total_batches) {
+    for i in 0..total_batches {
         let start = (i * batch_size) as i64;
         let rows = batch_size.min(cp - i * batch_size);
         let batch = make_batch(start, rows, dim);
-        let frag_id = memtable.insert(batch).await?;
-        memtable.mark_wal_flushed(&[frag_id], wal_pos + 1, &[i]);
+        let _frag_id = memtable.insert(batch).await?;
     }
 
     let temp_dir =
@@ -406,11 +406,19 @@ async fn measure_flush(
     let flusher = MemTableFlusher::new(store, base_path, uri, shard_id, manifest_store);
 
     // total_batches WAL entries were stamped at positions 1..=total_batches
-    // by the mark_wal_flushed loop above (1-based positions).
+    // by the loop above (1-based positions).
     let covered_wal_entry_position = total_batches as u64;
+    // Every batch was appended above, so the writer-global durable count is all of them.
+    let durable = total_batches;
     let t = Instant::now();
     let _result = flusher
-        .flush_with_indexes(&memtable, epoch, index_configs, covered_wal_entry_position)
+        .flush_with_indexes(
+            &memtable,
+            epoch,
+            index_configs,
+            covered_wal_entry_position,
+            durable,
+        )
         .await?;
     let elapsed = t.elapsed();
 
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
index 632bd37626c..e5bd050fd65 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
@@ -6,7 +6,7 @@
 //!
 //! Uses real embeddings from the `lance-format/fineweb-edu` HuggingFace
 //! dataset (384-dim) with an IVF-RQ index on the base table, then ingests
-//! additional rows through ShardWriter to populate flushed generations and
+//! additional rows through ShardWriter to populate SSTables and
 //! an active memtable.
 //!
 //! Three phases, selected with `--phase`:
@@ -102,7 +102,7 @@ struct Args {
     uri: String,
     base_rows: usize,
     max_memtable_rows: usize,
-    flushed_generations: usize,
+    sstables: usize,
     batch_rows: usize,
     queries: usize,
     k: usize,
@@ -120,7 +120,7 @@ impl Default for Args {
             uri: String::new(),
             base_rows: 1_000_000,
             max_memtable_rows: 100_000,
-            flushed_generations: 2,
+            sstables: 2,
             batch_rows: 1_000,
             queries: 100,
             k: 10,
@@ -166,7 +166,7 @@ fn parse_args() -> Result<Args> {
             }
             "--base-rows" => args.base_rows = parse_val(&flag, &value)?,
             "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?,
-            "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?,
+            "--sstables" => args.sstables = parse_val(&flag, &value)?,
             "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?,
             "--queries" => args.queries = parse_val(&flag, &value)?,
             "--k" => args.k = parse_val(&flag, &value)?,
@@ -520,7 +520,6 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: args.max_memtable_rows * row_bytes * 2,
         max_memtable_rows: args.max_memtable_rows,
         max_unflushed_memtable_bytes: args.max_memtable_rows * row_bytes * 6,
@@ -535,11 +534,11 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         Duration::from_millis(500)
     };
 
-    // Ingest N flushed generations + 1 active (50% full)
-    let num_flushed_target = args.flushed_generations;
+    // Ingest N SSTables + 1 active (50% full)
+    let num_sstable_target = args.sstables;
     let active_rows = args.max_memtable_rows / 2;
-    let total_memtable_rows = num_flushed_target * args.max_memtable_rows + active_rows;
-    let mut gen_sizes: Vec<usize> = (0..num_flushed_target)
+    let total_memtable_rows = num_sstable_target * args.max_memtable_rows + active_rows;
+    let mut gen_sizes: Vec<usize> = (0..num_sstable_target)
         .map(|_| args.max_memtable_rows)
         .collect();
     gen_sizes.push(active_rows);
@@ -616,14 +615,14 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
             gen_rows,
             ingest_start.elapsed().as_secs_f64(),
         );
-        if gen_idx < num_flushed_target {
+        if gen_idx < num_sstable_target {
             tokio::time::sleep(flush_wait).await;
         }
     }
     println!(
-        "ingested {} total memtable rows ({} flushed + active) in {:.1}s",
+        "ingested {} total memtable rows ({} SSTables + active) in {:.1}s",
         total_memtable_rows,
-        num_flushed_target,
+        num_sstable_target,
         ingest_start.elapsed().as_secs_f64(),
     );
 
@@ -633,17 +632,14 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
     let mut shard_snapshot = ShardSnapshot::new(shard_id);
     if let Some(ref m) = manifest {
         shard_snapshot = shard_snapshot.with_current_generation(m.current_generation);
-        for fg in &m.flushed_generations {
-            shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        for sstable in &m.sstables {
+            shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone());
         }
     }
-    let num_flushed = manifest
-        .as_ref()
-        .map(|m| m.flushed_generations.len())
-        .unwrap_or(0);
+    let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0);
     println!(
-        "manifest: {} flushed generations, current_generation={}",
-        num_flushed,
+        "manifest: {} SSTables, current_generation={}",
+        num_sstables,
         manifest.as_ref().map(|m| m.current_generation).unwrap_or(0)
     );
 
@@ -761,7 +757,7 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         "phase": "search",
         "base_rows": args.base_rows,
         "max_memtable_rows": args.max_memtable_rows,
-        "flushed_generations": num_flushed,
+        "sstables": num_sstables,
         "active_rows": active_rows,
         "vector_dim": VECTOR_DIM,
         "k": args.k,
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
index 14ec406e5e5..6fda0748006 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
@@ -179,7 +179,6 @@ async fn populate_shard_wal(
     let dataset = Dataset::open(dataset_uri).await.unwrap();
     let mut config = ShardWriterConfig::new(shard_id);
     config.durable_write = true;
-    config.sync_indexed_write = false;
     let writer = dataset.mem_wal_writer(shard_id, config).await.unwrap();
 
     for i in 0..num_entries {
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
index f4e4fb47395..79298daa872 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
@@ -99,10 +99,6 @@ impl Mode {
     fn durable_write(self) -> bool {
         matches!(self, Self::SyncNoIndex | Self::SyncIndexed)
     }
-
-    fn sync_indexed_write(self) -> bool {
-        matches!(self, Self::SyncIndexed)
-    }
 }
 
 /// Which index the MemTable maintains in the indexed (`*_idx`) modes.
@@ -192,7 +188,6 @@ struct Args {
     max_memtable_batches: Option<usize>,
     max_wal_buffer_size: usize,
     max_wal_flush_interval_ms: u64,
-    async_index_buffer_rows: usize,
     sample_interval_ms: u64,
     target_rows_per_sec: Option<f64>,
     num_partitions: usize,
@@ -223,7 +218,6 @@ impl Default for Args {
             max_memtable_batches: None,
             max_wal_buffer_size: 10 * 1024 * 1024,
             max_wal_flush_interval_ms: 100,
-            async_index_buffer_rows: 10_000,
             sample_interval_ms: 500,
             target_rows_per_sec: None,
             num_partitions: 1,
@@ -343,11 +337,9 @@ async fn run(args: Args) -> Result<()> {
     let shard_id = Uuid::new_v4();
     let mut config = ShardWriterConfig::new(shard_id)
         .with_durable_write(args.mode.durable_write())
-        .with_sync_indexed_write(args.mode.sync_indexed_write())
         .with_max_memtable_size(args.max_memtable_size)
         .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes)
         .with_max_wal_buffer_size(args.max_wal_buffer_size)
-        .with_async_index_buffer_rows(args.async_index_buffer_rows)
         .with_max_memtable_rows(memtable_limits.rows)
         .with_max_memtable_batches(memtable_limits.batches);
     if args.max_wal_flush_interval_ms == 0 {
@@ -544,7 +536,6 @@ async fn run(args: Args) -> Result<()> {
         "max_memtable_batches": memtable_limits.batches,
         "max_wal_buffer_size": args.max_wal_buffer_size,
         "max_wal_flush_interval_ms": args.max_wal_flush_interval_ms,
-        "async_index_buffer_rows": args.async_index_buffer_rows,
         "sample_interval_ms": args.sample_interval_ms,
         "skip_close": args.skip_close,
         "setup_seconds": setup_s,
@@ -650,7 +641,7 @@ fn push_sample(
         "active_memtable_bytes": memtable.as_ref().map(|stats| stats.estimated_size),
         "active_memtable_generation": memtable.as_ref().map(|stats| stats.generation),
         "active_memtable_max_buffered_batch_position": memtable.as_ref().and_then(|stats| stats.max_buffered_batch_position),
-        "active_memtable_max_flushed_batch_position": memtable.as_ref().and_then(|stats| stats.max_flushed_batch_position),
+        "active_memtable_durable_batch_count": memtable.as_ref().map(|stats| stats.durable_batch_count),
         "wal_queue_pending_batches": memtable.as_ref().map(|stats| stats.pending_wal_batch_count),
         "wal_queue_pending_rows": memtable.as_ref().map(|stats| stats.pending_wal_row_count),
         "wal_queue_pending_bytes": memtable.as_ref().map(|stats| stats.pending_wal_estimated_bytes),
@@ -667,7 +658,7 @@ fn memtable_stats_json(memtable: Option<&MemTableStats>) -> serde_json::Value {
             "estimated_size": stats.estimated_size,
             "generation": stats.generation,
             "max_buffered_batch_position": stats.max_buffered_batch_position,
-            "max_flushed_batch_position": stats.max_flushed_batch_position,
+            "durable_batch_count": stats.durable_batch_count,
             "wal_queue_pending_start_batch_position": stats.pending_wal_start_batch_position,
             "wal_queue_pending_end_batch_position": stats.pending_wal_end_batch_position,
             "wal_queue_pending_batches": stats.pending_wal_batch_count,
@@ -980,7 +971,6 @@ fn parse_args() -> Result<Args> {
             "--max-wal-flush-interval-ms" => {
                 args.max_wal_flush_interval_ms = parse(&flag, &value)?;
             }
-            "--async-index-buffer-rows" => args.async_index_buffer_rows = parse(&flag, &value)?,
             "--sample-interval-ms" => args.sample_interval_ms = parse(&flag, &value)?,
             "--target-rows-per-sec" => args.target_rows_per_sec = Some(parse(&flag, &value)?),
             "--num-partitions" => args.num_partitions = parse(&flag, &value)?,
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_write.rs b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
index 6d3a31c4011..b5018fe09f1 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_write.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
@@ -30,7 +30,6 @@
 //! - `BATCH_SIZE`: Number of rows per write batch (default: 20)
 //! - `NUM_BATCHES`: Total number of batches to write (default: 1000)
 //! - `DURABLE_WRITE`: yes/no/both (default: no) - whether writes wait for WAL flush
-//! - `INDEXED_WRITE`: yes/no/both (default: no) - whether writes update indexes synchronously
 //! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from ShardWriterConfig)
 //! - `MAX_FLUSH_INTERVAL_MS`: WAL flush interval in milliseconds, 0 to disable (default: 1000ms)
 //! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from ShardWriterConfig)
@@ -44,8 +43,7 @@
 //!   `no` uses WAL-only mode (no MemTable, no indexes, no Lance flushes; pure WAL throughput).
 //!   `both` runs each combination twice, once per mode, side-by-side.
 //!   When `no` or `both`, the WAL-only branch always runs with
-//!   `MEMWAL_MAINTAINED_INDEXES=none` and skips `INDEXED_WRITE=yes`
-//!   (sync-indexed writes require a MemTable).
+//!   `MEMWAL_MAINTAINED_INDEXES=none`.
 //! - `SAMPLE_SIZE`: Number of benchmark iterations (default: 10, minimum: 10)
 
 #![allow(clippy::print_stdout, clippy::print_stderr)]
@@ -119,11 +117,6 @@ fn get_durable_write_options() -> Vec<bool> {
     parse_yes_no_both("DURABLE_WRITE", "no")
 }
 
-/// Get indexed write settings from environment.
-fn get_indexed_write_options() -> Vec<bool> {
-    parse_yes_no_both("INDEXED_WRITE", "no")
-}
-
 /// Get enable_memtable settings from environment. Default `yes` keeps
 /// existing benchmark behavior; `no` runs WAL-only mode; `both` runs both
 /// modes side-by-side for comparison.
@@ -442,31 +435,26 @@ fn build_label(
     num_batches: usize,
     batch_size: usize,
     durable: bool,
-    indexed: bool,
     enable_memtable: bool,
     storage: &str,
 ) -> String {
     let durable_str = if durable { "durable" } else { "nondurable" };
-    // sync_indexed_write controls sync vs async index updates
-    let indexed_str = if indexed { "sync_idx" } else { "async_idx" };
     let mode_str = if enable_memtable {
         "memtable"
     } else {
         "wal_only"
     };
     format!(
-        "{}x{} {} {} {} ({})",
-        num_batches, batch_size, mode_str, durable_str, indexed_str, storage
+        "{}x{} {} {} ({})",
+        num_batches, batch_size, mode_str, durable_str, storage
     )
 }
 
 /// Build dataset name prefix from config options.
-fn build_name_prefix(durable: bool, indexed: bool, enable_memtable: bool) -> String {
+fn build_name_prefix(durable: bool, enable_memtable: bool) -> String {
     let d = if durable { "d" } else { "nd" };
-    // sync_indexed_write: sync (si) vs async (ai)
-    let i = if indexed { "si" } else { "ai" };
     let m = if enable_memtable { "mt" } else { "wo" };
-    format!("{}_{}_{}", m, d, i)
+    format!("{}_{}", m, d)
 }
 
 /// Benchmark Lance MemWAL write throughput.
@@ -490,7 +478,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
     let maintained_indexes = get_maintained_indexes();
 
     let durable_options = get_durable_write_options();
-    let indexed_options = get_indexed_write_options();
     let enable_memtable_options = get_enable_memtable_options();
     let max_wal_buffer_size = get_max_wal_buffer_size();
     let max_flush_interval = get_max_flush_interval();
@@ -550,70 +537,56 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
     // Generate benchmarks for all combinations
     for &enable_memtable in &enable_memtable_options {
         for &durable in &durable_options {
-            for &indexed in &indexed_options {
-                if !enable_memtable && indexed {
-                    eprintln!(
-                        "Skipping wal_only + sync_idx (sync_indexed_write requires a MemTable)"
-                    );
-                    continue;
-                }
-
-                let label = build_label(
-                    num_batches,
-                    batch_size,
-                    durable,
-                    indexed,
-                    enable_memtable,
-                    storage_label,
-                );
-                let name_prefix = build_name_prefix(durable, indexed, enable_memtable);
-
-                // WAL-only mode never uses indexes; force the dataset
-                // setup to skip the MemWAL index list.
-                let effective_indexes: Vec<String> = if enable_memtable {
-                    maintained_indexes.clone()
-                } else {
-                    Vec::new()
-                };
-
-                // Create dataset ONCE before benchmark iterations
-                // Each iteration will use a different shard on the same dataset
-                let dataset = rt.block_on(create_dataset(
-                    &schema,
-                    &name_prefix,
-                    vector_dim,
-                    &effective_indexes,
-                    &dataset_prefix,
-                ));
-                let dataset_uri = dataset.uri().to_string();
-
-                // Pre-generate all batches before timing (outside iter_custom)
-                let batches: Arc<Vec<RecordBatch>> = Arc::new(
-                    (0..num_batches)
-                        .map(|i| {
-                            create_test_batch(
-                                &schema,
-                                (i * batch_size) as i64,
-                                batch_size,
-                                vector_dim,
-                            )
-                        })
-                        .collect(),
-                );
-
-                println!("Running: {}", label);
-
-                // Track if we've printed stats (only print once across all samples)
-                let stats_printed = Arc::new(AtomicBool::new(false));
-
-                group.bench_with_input(
-                    BenchmarkId::new("Lance MemWAL", &label),
-                    &(batch_size, num_batches, durable, indexed, row_size_bytes),
-                    |b, &(_batch_size, _num_batches, durable, indexed, row_size_bytes)| {
-                        let dataset_uri = dataset_uri.clone();
-                        let batches = batches.clone();
-                        let stats_printed = stats_printed.clone();
-                        b.to_async(&rt).iter_custom(|iters| {
+            let label = build_label(
+                num_batches,
+                batch_size,
+                durable,
+                enable_memtable,
+                storage_label,
+            );
+            let name_prefix = build_name_prefix(durable, enable_memtable);
+
+            // WAL-only mode never uses indexes; force the dataset
+            // setup to skip the MemWAL index list.
+            let effective_indexes: Vec<String> = if enable_memtable {
+                maintained_indexes.clone()
+            } else {
+                Vec::new()
+            };
+
+            // Create dataset ONCE before benchmark iterations
+            // Each iteration will use a different shard on the same dataset
+            let dataset = rt.block_on(create_dataset(
+                &schema,
+                &name_prefix,
+                vector_dim,
+                &effective_indexes,
+                &dataset_prefix,
+            ));
+            let dataset_uri = dataset.uri().to_string();
+
+            // Pre-generate all batches before timing (outside iter_custom)
+            let batches: Arc<Vec<RecordBatch>> = Arc::new(
+                (0..num_batches)
+                    .map(|i| {
+                        create_test_batch(&schema, (i * batch_size) as i64, batch_size, vector_dim)
+                    })
+                    .collect(),
+            );
+
+            println!("Running: {}", label);
+
+            // Track if we've printed stats (only print once across all samples)
+            let stats_printed = Arc::new(AtomicBool::new(false));
+
+            group.bench_with_input(
+                BenchmarkId::new("Lance MemWAL", &label),
+                &(batch_size, num_batches, durable, row_size_bytes),
+                |b, &(_batch_size, _num_batches, durable, row_size_bytes)| {
+                    let dataset_uri = dataset_uri.clone();
+                    let batches = batches.clone();
+                    let stats_printed = stats_printed.clone();
+                    b.to_async(&rt).iter_custom(|iters| {
                         let dataset_uri = dataset_uri.clone();
                         let batches = batches.clone();
                         let stats_printed = stats_printed.clone();
@@ -631,10 +604,10 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                     shard_id,
                                     shard_spec_id: 0,
                                     max_wal_persist_retries: 3,
-                                    wal_persist_retry_base_delay:
-                                        std::time::Duration::from_millis(50),
+                                    wal_persist_retry_base_delay: std::time::Duration::from_millis(
+                                        50,
+                                    ),
                                     durable_write: durable,
-                                    sync_indexed_write: indexed,
                                     max_wal_buffer_size: max_wal_buffer_size
                                         .unwrap_or(default_config.max_wal_buffer_size),
                                     max_wal_flush_interval: max_flush_interval
@@ -643,8 +616,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                         .unwrap_or(default_config.max_memtable_size),
                                     max_memtable_rows: default_config.max_memtable_rows,
                                     max_memtable_batches: default_config.max_memtable_batches,
-                                    async_index_buffer_rows: default_config.async_index_buffer_rows,
-                                    async_index_interval: default_config.async_index_interval,
                                     manifest_scan_batch_size: default_config
                                         .manifest_scan_batch_size,
                                     max_unflushed_memtable_bytes: default_config
@@ -656,6 +627,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                     enable_memtable,
                                     hnsw_params: default_config.hnsw_params,
                                     warmer: None,
+                                    store_params: default_config.store_params,
+                                    session: default_config.session,
                                 };
 
                                 // Get writer through Dataset API (index configs loaded automatically)
@@ -704,9 +677,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                             total_duration
                         }
                     })
-                    },
-                );
-            }
+                },
+            );
         }
     }
 
diff --git a/rust/lance/benches/s3_file_reader_diagnostics.rs b/rust/lance/benches/s3_file_reader_diagnostics.rs
new file mode 100644
index 00000000000..1762048d495
--- /dev/null
+++ b/rust/lance/benches/s3_file_reader_diagnostics.rs
@@ -0,0 +1,2357 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#![allow(clippy::print_stdout)]
+#![recursion_limit = "256"]
+
+use std::collections::BTreeMap;
+use std::env;
+use std::fs;
+use std::hint::black_box;
+use std::ops::Range;
+use std::sync::{
+    Arc, Mutex,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use arrow_array::RecordBatch;
+use futures::future::BoxFuture;
+use futures::stream::{FuturesOrdered, FuturesUnordered};
+use futures::{FutureExt, StreamExt, TryStreamExt};
+use lance::dataset::ProjectionRequest;
+use lance::dataset::builder::DatasetBuilder;
+use lance::dataset::fragment::{FileFragment, FragReadConfig};
+use lance::dataset::scanner::{ExecutionStatsCallback, ExecutionSummaryCounts};
+use lance_core::datatypes::Schema;
+use lance_encoding::decoder::PageEncoding;
+use lance_encoding::format::pb21;
+use lance_file::reader::{
+    DEFAULT_READ_CHUNK_SIZE, FileReader as LanceFileReader, FileReaderOptions,
+};
+use lance_io::object_store::ObjectStore as LanceObjectStore;
+use lance_io::scheduler::{FileScheduler, ScanScheduler, ScanStats, SchedulerConfig};
+use lance_io::utils::CachedFileSize;
+use serde_json::{Value, json};
+use tracing::field::{Field, Visit};
+use tracing::subscriber::Interest;
+use tracing::{Event, Metadata, Subscriber};
+use tracing_subscriber::layer::{Context, Layer};
+use tracing_subscriber::prelude::*;
+
+type Error = Box<dyn std::error::Error + Send + Sync>;
+type Result<T> = std::result::Result<T, Error>;
+
+const GIB: u64 = 1024 * 1024 * 1024;
+const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state";
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum SchedulerQueueKind {
+    Standard,
+    Lite,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct SchedulerDiagnostics {
+    kind: SchedulerQueueKind,
+    stats: ScanStats,
+    io_capacity: u64,
+    iops_available: u64,
+    active_iops: u64,
+    pending_iops: u64,
+    pending_bytes: u64,
+    bytes_available: i64,
+    bytes_reserved: i64,
+    io_buffer_size_bytes: u64,
+    priorities_in_flight: u64,
+    no_backpressure: bool,
+    head_task_bytes: Option<u64>,
+    head_task_priority_high: Option<u64>,
+    head_task_priority_low: Option<u64>,
+    min_in_flight_priority_high: Option<u64>,
+    min_in_flight_priority_low: Option<u64>,
+    head_task_can_deliver: Option<bool>,
+    head_task_priority_bypass: Option<bool>,
+    head_task_blocked_by_iops: Option<bool>,
+    head_task_blocked_by_bytes: Option<bool>,
+}
+
+#[derive(Debug, Clone)]
+struct Config {
+    backend: Backend,
+    uri: String,
+    dataset_version: u64,
+    columns: Option<Vec<String>>,
+    limit_rows: u64,
+    target_bytes: Option<u64>,
+    raw_range_size_bytes: u64,
+    raw_range_mode: RawRangeMode,
+    raw_column_indices: Option<Vec<u32>>,
+    raw_submit_mode: RawSubmitMode,
+    raw_completion_mode: RawCompletionMode,
+    take_repetitions: u64,
+    io_buffer_gib: Vec<Option<u64>>,
+    batch_size: u32,
+    batch_size_bytes: Option<u64>,
+    skip_batch_byte_accounting: bool,
+    read_chunk_size: Option<u64>,
+    fragment_concurrency: usize,
+    batch_concurrency: usize,
+    sample_ms: u64,
+    out_dir: String,
+    case_name: String,
+    describe_layout: bool,
+    detach_fragment_streams: bool,
+    drop_read_tasks: bool,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum Backend {
+    FileReader,
+    Scanner,
+    SchedulerRaw,
+    DatasetTake,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawSubmitMode {
+    Single,
+    SplitNoConcat,
+    SplitConcat,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawRangeMode {
+    FileSequential,
+    MetadataPages,
+    MetadataPagesRoundRobin,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawCompletionMode {
+    Unordered,
+    Ordered,
+}
+
+impl RawRangeMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::FileSequential => "file-sequential",
+            Self::MetadataPages => "metadata-pages",
+            Self::MetadataPagesRoundRobin => "metadata-pages-round-robin",
+        }
+    }
+}
+
+impl RawSubmitMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Single => "single",
+            Self::SplitNoConcat => "split-no-concat",
+            Self::SplitConcat => "split-concat",
+        }
+    }
+}
+
+impl RawCompletionMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Unordered => "unordered",
+            Self::Ordered => "ordered",
+        }
+    }
+}
+
+impl Backend {
+    fn name(self) -> &'static str {
+        match self {
+            Self::FileReader => "lance-file-reader",
+            Self::Scanner => "lance-scanner",
+            Self::SchedulerRaw => "lance-scheduler-raw",
+            Self::DatasetTake => "lance-dataset-take",
+        }
+    }
+
+    fn layer(self) -> &'static str {
+        match self {
+            Self::FileReader => "file-reader",
+            Self::Scanner => "scanner",
+            Self::SchedulerRaw => "scheduler",
+            Self::DatasetTake => "dataset-take",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct CpuSample {
+    idle: u64,
+    total: u64,
+}
+
+#[derive(Debug, Default)]
+struct SharedCounters {
+    fragments_started: AtomicU64,
+    fragments_completed: AtomicU64,
+    batch_futures_emitted: AtomicU64,
+    batch_futures_received: AtomicU64,
+    batches_completed: AtomicU64,
+    rows_completed: AtomicU64,
+    arrow_bytes: AtomicU64,
+    open_reader_ns: AtomicU64,
+    read_stream_create_ns: AtomicU64,
+    next_batch_poll_ns: AtomicU64,
+    channel_send_wait_ns: AtomicU64,
+    decode_ns: AtomicU64,
+    raw_reassemble_ns: AtomicU64,
+}
+
+#[derive(Debug)]
+struct CaseStats {
+    rows: u64,
+    batches: u64,
+    arrow_bytes: u64,
+    planned_fragments: usize,
+    planned_rows: u64,
+    elapsed: Duration,
+    producer_finished_at: Option<Duration>,
+    peak_decode_in_flight: usize,
+    cpu_avg: Option<f64>,
+    scheduler_diagnostics: SchedulerDiagnostics,
+    counters: Arc<SharedCounters>,
+    samples: Vec<Value>,
+}
+
+#[derive(Debug)]
+struct LastSample {
+    elapsed: Duration,
+    scheduler_stats: ScanStats,
+    rows: u64,
+    batches: u64,
+    arrow_bytes: u64,
+}
+
+fn usage() -> &'static str {
+    "usage: s3_file_reader_diagnostics --uri <s3-dataset-uri> \
+     [--backend <file-reader|scanner|scheduler-raw>] \
+     [--dataset-version <n>] [--columns <all|col[,col...]>] \
+     [--limit-rows <n>] [--target-bytes <n>] [--raw-range-size-bytes <n>] \
+     [--raw-range-mode <file-sequential|metadata-pages|metadata-pages-round-robin>] \
+     [--raw-column-indices <all|n[,n...]>] \
+     [--raw-submit-mode <single|split-no-concat|split-concat>] \
+     [--raw-completion-mode <unordered|ordered>] \
+     [--take-repetitions <n>] \
+     [--io-buffer-gib <auto|n[,n...]>] \
+     [--batch-size <n>] [--batch-size-bytes <n>] [--read-chunk-size <n>] \
+     [--skip-batch-byte-accounting] \
+     [--fragment-concurrency <n>] [--batch-concurrency <n>] \
+     [--sample-ms <n>] [--out-dir <path>] [--case <name>] \
+     [--detach-fragment-streams] [--drop-read-tasks] [--describe-layout]"
+}
+
+fn parse_args() -> Result<Config> {
+    let mut backend = Backend::FileReader;
+    let mut uri = None;
+    let mut dataset_version = 1u64;
+    let mut columns = Some(vec!["vector".to_string()]);
+    let mut limit_rows = 67_108_864u64;
+    let mut target_bytes = None;
+    let mut raw_range_size_bytes = 16 * 1024 * 1024;
+    let mut raw_range_mode = RawRangeMode::FileSequential;
+    let mut raw_column_indices = None;
+    let mut raw_submit_mode = RawSubmitMode::Single;
+    let mut raw_completion_mode = RawCompletionMode::Unordered;
+    let mut take_repetitions = 100u64;
+    let mut io_buffer_gib = vec![Some(8)];
+    let mut batch_size = 8192u32;
+    let mut batch_size_bytes = None;
+    let mut skip_batch_byte_accounting = false;
+    let mut read_chunk_size = None;
+    let mut fragment_concurrency = 256usize;
+    let mut batch_concurrency = 256usize;
+    let mut sample_ms = 1000u64;
+    let mut out_dir = "/tmp/lance-s3-bottleneck-results".to_string();
+    let mut case_name = "lance-file-reader-diagnostics".to_string();
+    let mut describe_layout = false;
+    let mut detach_fragment_streams = false;
+    let mut drop_read_tasks = false;
+
+    let mut args = env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--backend" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --backend. {}", usage()))?;
+                backend = parse_backend(&value)?;
+            }
+            "--uri" => uri = args.next(),
+            "--dataset-version" => {
+                dataset_version = parse_required_value(&mut args, "--dataset-version")?;
+            }
+            "--columns" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --columns. {}", usage()))?;
+                columns = parse_columns(&value)?;
+            }
+            "--limit-rows" => {
+                limit_rows = parse_required_value(&mut args, "--limit-rows")?;
+            }
+            "--target-bytes" => {
+                target_bytes = Some(parse_required_value(&mut args, "--target-bytes")?);
+            }
+            "--raw-range-size-bytes" => {
+                raw_range_size_bytes = parse_required_value(&mut args, "--raw-range-size-bytes")?;
+            }
+            "--raw-range-mode" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --raw-range-mode. {}", usage()))?;
+                raw_range_mode = parse_raw_range_mode(&value)?;
+            }
+            "--raw-column-indices" => {
+                let value = args.next().ok_or_else(|| {
+                    format!("missing value for --raw-column-indices. {}", usage())
+                })?;
+                raw_column_indices = parse_raw_column_indices(&value)?;
+            }
+            "--raw-submit-mode" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --raw-submit-mode. {}", usage()))?;
+                raw_submit_mode = parse_raw_submit_mode(&value)?;
+            }
+            "--raw-completion-mode" => {
+                let value = args.next().ok_or_else(|| {
+                    format!("missing value for --raw-completion-mode. {}", usage())
+                })?;
+                raw_completion_mode = parse_raw_completion_mode(&value)?;
+            }
+            "--take-repetitions" => {
+                take_repetitions = parse_required_value(&mut args, "--take-repetitions")?;
+            }
+            "--io-buffer-gib" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --io-buffer-gib. {}", usage()))?;
+                io_buffer_gib = parse_io_buffer_gib(&value)?;
+            }
+            "--batch-size" => {
+                batch_size = parse_required_value(&mut args, "--batch-size")?;
+            }
+            "--batch-size-bytes" => {
+                batch_size_bytes = Some(parse_required_value(&mut args, "--batch-size-bytes")?);
+            }
+            "--skip-batch-byte-accounting" => {
+                skip_batch_byte_accounting = true;
+            }
+            "--read-chunk-size" => {
+                read_chunk_size = Some(parse_required_value(&mut args, "--read-chunk-size")?);
+            }
+            "--fragment-concurrency" => {
+                fragment_concurrency = parse_required_value(&mut args, "--fragment-concurrency")?;
+            }
+            "--batch-concurrency" => {
+                batch_concurrency = parse_required_value(&mut args, "--batch-concurrency")?;
+            }
+            "--sample-ms" => {
+                sample_ms = parse_required_value(&mut args, "--sample-ms")?;
+            }
+            "--out-dir" => {
+                out_dir = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --out-dir. {}", usage()))?;
+            }
+            "--case" => {
+                case_name = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --case. {}", usage()))?;
+            }
+            "--describe-layout" => {
+                describe_layout = true;
+            }
+            "--detach-fragment-streams" => {
+                detach_fragment_streams = true;
+            }
+            "--drop-read-tasks" => {
+                drop_read_tasks = true;
+            }
+            "--help" | "-h" => {
+                println!("{}", usage());
+                std::process::exit(0);
+            }
+            "--bench" => {
+                // Cargo appends this flag when running harness-free benches.
+            }
+            other => {
+                return Err(format!("unknown argument {other}. {}", usage()).into());
+            }
+        }
+    }
+
+    let uri = uri.ok_or_else(|| format!("missing required --uri. {}", usage()))?;
+    if limit_rows == 0 {
+        return Err("--limit-rows must be greater than zero".into());
+    }
+    if matches!(target_bytes, Some(0)) {
+        return Err("--target-bytes must be greater than zero".into());
+    }
+    if raw_range_size_bytes == 0 {
+        return Err("--raw-range-size-bytes must be greater than zero".into());
+    }
+    if take_repetitions == 0 {
+        return Err("--take-repetitions must be greater than zero".into());
+    }
+    if io_buffer_gib.is_empty() {
+        return Err("--io-buffer-gib must not be empty".into());
+    }
+    if batch_size == 0 {
+        return Err("--batch-size must be greater than zero".into());
+    }
+    if matches!(batch_size_bytes, Some(0)) {
+        return Err("--batch-size-bytes must be greater than zero".into());
+    }
+    if matches!(read_chunk_size, Some(0)) {
+        return Err("--read-chunk-size must be greater than zero".into());
+    }
+    if fragment_concurrency == 0 && !matches!(backend, Backend::Scanner) {
+        return Err("--fragment-concurrency must be greater than zero".into());
+    }
+    if batch_concurrency == 0 && !matches!(backend, Backend::Scanner) {
+        return Err("--batch-concurrency must be greater than zero".into());
+    }
+    if sample_ms == 0 {
+        return Err("--sample-ms must be greater than zero".into());
+    }
+
+    Ok(Config {
+        backend,
+        uri,
+        dataset_version,
+        columns,
+        limit_rows,
+        target_bytes,
+        raw_range_size_bytes,
+        raw_range_mode,
+        raw_column_indices,
+        raw_submit_mode,
+        raw_completion_mode,
+        take_repetitions,
+        io_buffer_gib,
+        batch_size,
+        batch_size_bytes,
+        skip_batch_byte_accounting,
+        read_chunk_size,
+        fragment_concurrency,
+        batch_concurrency,
+        sample_ms,
+        out_dir,
+        case_name,
+        describe_layout,
+        detach_fragment_streams,
+        drop_read_tasks,
+    })
+}
+
+fn parse_backend(value: &str) -> Result<Backend> {
+    match value {
+        "file-reader" | "lance-file-reader" => Ok(Backend::FileReader),
+        "scanner" | "lance-scanner" => Ok(Backend::Scanner),
+        "scheduler-raw" | "lance-scheduler-raw" => Ok(Backend::SchedulerRaw),
+        "dataset-take" | "take" | "lance-dataset-take" => Ok(Backend::DatasetTake),
+        other => Err(format!(
+            "invalid --backend value {other}; expected file-reader, scanner, scheduler-raw, or dataset-take"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_submit_mode(value: &str) -> Result<RawSubmitMode> {
+    match value {
+        "single" => Ok(RawSubmitMode::Single),
+        "split-no-concat" => Ok(RawSubmitMode::SplitNoConcat),
+        "split-concat" => Ok(RawSubmitMode::SplitConcat),
+        other => Err(format!(
+            "invalid --raw-submit-mode value {other}; expected single, split-no-concat, or split-concat"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_range_mode(value: &str) -> Result<RawRangeMode> {
+    match value {
+        "file-sequential" => Ok(RawRangeMode::FileSequential),
+        "metadata-pages" => Ok(RawRangeMode::MetadataPages),
+        "metadata-pages-round-robin" => Ok(RawRangeMode::MetadataPagesRoundRobin),
+        other => Err(format!(
+            "invalid --raw-range-mode value {other}; expected file-sequential, metadata-pages, or metadata-pages-round-robin"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_column_indices(value: &str) -> Result<Option<Vec<u32>>> {
+    if value == "all" {
+        return Ok(None);
+    }
+
+    let indices = value
+        .split(',')
+        .map(str::trim)
+        .filter(|part| !part.is_empty())
+        .map(|part| {
+            part.parse::<u32>()
+                .map_err(|err| format!("invalid raw column index {part}: {err}").into())
+        })
+        .collect::<Result<Vec<_>>>()?;
+    if indices.is_empty() {
+        return Err("--raw-column-indices must specify at least one column index or all".into());
+    }
+    Ok(Some(indices))
+}
+
+fn parse_raw_completion_mode(value: &str) -> Result<RawCompletionMode> {
+    match value {
+        "unordered" => Ok(RawCompletionMode::Unordered),
+        "ordered" => Ok(RawCompletionMode::Ordered),
+        other => Err(format!(
+            "invalid --raw-completion-mode value {other}; expected unordered or ordered"
+        )
+        .into()),
+    }
+}
+
+fn parse_required_value<T>(args: &mut impl Iterator<Item = String>, name: &str) -> Result<T>
+where
+    T: std::str::FromStr,
+    T::Err: std::fmt::Display + Send + Sync + 'static,
+{
+    let value = args
+        .next()
+        .ok_or_else(|| format!("missing value for {name}. {}", usage()))?;
+    value
+        .parse()
+        .map_err(|err| format!("invalid {name} value {value}: {err}").into())
+}
+
+fn parse_columns(value: &str) -> Result<Option<Vec<String>>> {
+    match value {
+        "all" => Ok(None),
+        "empty" => Err("FileReader benchmark requires at least one data column".into()),
+        _ => Ok(Some(
+            value
+                .split(',')
+                .map(str::trim)
+                .filter(|column| !column.is_empty())
+                .map(ToString::to_string)
+                .collect(),
+        )),
+    }
+}
+
+fn parse_io_buffer_gib(value: &str) -> Result<Vec<Option<u64>>> {
+    value
+        .split(',')
+        .map(|part| {
+            let part = part.trim();
+            if part == "auto" {
+                Ok(None)
+            } else {
+                part.parse::<u64>()
+                    .map(Some)
+                    .map_err(|err| format!("invalid --io-buffer-gib value {part}: {err}").into())
+            }
+        })
+        .collect()
+}
+
+fn projection_name(columns: &Option<Vec<String>>) -> String {
+    match columns {
+        None => "all".to_string(),
+        Some(columns) => columns.join(","),
+    }
+}
+
+fn page_layout_kind(encoding: &PageEncoding) -> &'static str {
+    match encoding {
+        PageEncoding::Legacy(_) => "legacy",
+        PageEncoding::Structural(layout) => match layout.layout.as_ref() {
+            Some(pb21::page_layout::Layout::MiniBlockLayout(_)) => "miniblock",
+            Some(pb21::page_layout::Layout::ConstantLayout(_)) => "constant",
+            Some(pb21::page_layout::Layout::FullZipLayout(_)) => "fullzip",
+            Some(pb21::page_layout::Layout::BlobLayout(_)) => "blob",
+            Some(pb21::page_layout::Layout::SparseLayout(_)) => "sparse",
+            None => "missing",
+        },
+    }
+}
+
+fn summarize_u64(values: &[u64]) -> Value {
+    if values.is_empty() {
+        return json!({
+            "count": 0,
+            "min": null,
+            "p50": null,
+            "p90": null,
+            "max": null,
+            "sum": 0,
+        });
+    }
+    let mut sorted = values.to_vec();
+    sorted.sort_unstable();
+    let percentile = |p: f64| {
+        let idx = ((sorted.len() - 1) as f64 * p).round() as usize;
+        sorted[idx]
+    };
+    json!({
+        "count": sorted.len(),
+        "min": sorted[0],
+        "p50": percentile(0.5),
+        "p90": percentile(0.9),
+        "max": *sorted.last().unwrap(),
+        "sum": values.iter().sum::<u64>(),
+    })
+}
+
+async fn describe_layout(config: &Config) -> Result<()> {
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let fragment = dataset
+        .fragments()
+        .first()
+        .ok_or("dataset has no fragments")?;
+    let data_file = fragment
+        .files
+        .first()
+        .ok_or("first fragment has no data files")?;
+    if data_file.base_id.is_some() {
+        return Err("layout diagnostics do not support external base data files yet".into());
+    }
+
+    let data_path = dataset.data_dir().join(data_file.path.as_str());
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler = ScanScheduler::new(object_store, SchedulerConfig::new(8 * GIB));
+    let file_scheduler = scheduler
+        .open_file(&data_path, &CachedFileSize::unknown())
+        .await?;
+    let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?;
+
+    let columns = metadata
+        .column_infos
+        .iter()
+        .map(|column| {
+            let mut layout_counts = BTreeMap::new();
+            let mut page_rows = Vec::with_capacity(column.page_infos.len());
+            let mut page_bytes = Vec::with_capacity(column.page_infos.len());
+            for page in column.page_infos.iter() {
+                *layout_counts
+                    .entry(page_layout_kind(&page.encoding))
+                    .or_insert(0usize) += 1;
+                page_rows.push(page.num_rows);
+                page_bytes.push(
+                    page.buffer_offsets_and_sizes
+                        .iter()
+                        .map(|(_, size)| *size)
+                        .sum::<u64>(),
+                );
+            }
+            json!({
+                "column_index": column.index,
+                "num_pages": column.page_infos.len(),
+                "layout_counts": layout_counts,
+                "page_rows": summarize_u64(&page_rows),
+                "page_bytes": summarize_u64(&page_bytes),
+            })
+        })
+        .collect::<Vec<_>>();
+
+    println!(
+        "{}",
+        serde_json::to_string_pretty(&json!({
+            "dataset_uri": config.uri,
+            "dataset_version": config.dataset_version,
+            "fragment_id": fragment.id,
+            "data_file_path": data_file.path,
+            "resolved_data_path": data_path.to_string(),
+            "file_version": metadata.version().to_string(),
+            "num_rows": metadata.num_rows,
+            "num_data_bytes": metadata.num_data_bytes,
+            "columns": columns,
+        }))?
+    );
+    Ok(())
+}
+
+fn projected_schema(dataset_schema: &Schema, columns: &Option<Vec<String>>) -> Result<Schema> {
+    Ok(match columns {
+        None => dataset_schema.clone(),
+        Some(columns) => dataset_schema.project(columns)?,
+    })
+}
+
+fn file_reader_options(config: &Config) -> Option<FileReaderOptions> {
+    if config.batch_size_bytes.is_none() && config.read_chunk_size.is_none() {
+        return None;
+    }
+    Some(FileReaderOptions {
+        batch_size_bytes: config.batch_size_bytes,
+        read_chunk_size: config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE),
+        ..Default::default()
+    })
+}
+
+fn add_duration(counter: &AtomicU64, duration: Duration) {
+    let nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+    counter.fetch_add(nanos, Ordering::Relaxed);
+}
+
+fn ns_to_seconds(ns: u64) -> f64 {
+    ns as f64 / 1_000_000_000.0
+}
+
+fn diff_u64(current: u64, previous: u64) -> u64 {
+    current.saturating_sub(previous)
+}
+
+fn scheduler_kind_name(kind: SchedulerQueueKind) -> &'static str {
+    match kind {
+        SchedulerQueueKind::Standard => "standard",
+        SchedulerQueueKind::Lite => "lite",
+    }
+}
+
+fn diagnostics_json(diagnostics: SchedulerDiagnostics) -> Value {
+    json!({
+        "queue_kind": scheduler_kind_name(diagnostics.kind),
+        "scheduler_iops": diagnostics.stats.iops,
+        "scheduler_requests": diagnostics.stats.requests,
+        "scheduler_bytes_read": diagnostics.stats.bytes_read,
+        "io_capacity": diagnostics.io_capacity,
+        "iops_available": diagnostics.iops_available,
+        "active_iops": diagnostics.active_iops,
+        "pending_iops": diagnostics.pending_iops,
+        "pending_bytes": diagnostics.pending_bytes,
+        "bytes_available": diagnostics.bytes_available,
+        "bytes_reserved": diagnostics.bytes_reserved,
+        "io_buffer_size_bytes": diagnostics.io_buffer_size_bytes,
+        "priorities_in_flight": diagnostics.priorities_in_flight,
+        "no_backpressure": diagnostics.no_backpressure,
+        "head_task_bytes": diagnostics.head_task_bytes,
+        "head_task_priority_high": diagnostics.head_task_priority_high,
+        "head_task_priority_low": diagnostics.head_task_priority_low,
+        "min_in_flight_priority_high": diagnostics.min_in_flight_priority_high,
+        "min_in_flight_priority_low": diagnostics.min_in_flight_priority_low,
+        "head_task_can_deliver": diagnostics.head_task_can_deliver,
+        "head_task_priority_bypass": diagnostics.head_task_priority_bypass,
+        "head_task_blocked_by_iops": diagnostics.head_task_blocked_by_iops,
+        "head_task_blocked_by_bytes": diagnostics.head_task_blocked_by_bytes,
+    })
+}
+
+#[derive(Debug, Default)]
+struct ExecutionStatsHolder {
+    collected_stats: Arc<Mutex<Option<ExecutionSummaryCounts>>>,
+}
+
+impl ExecutionStatsHolder {
+    fn get_setter(&self) -> ExecutionStatsCallback {
+        let collected_stats = self.collected_stats.clone();
+        Arc::new(move |stats| {
+            *collected_stats.lock().unwrap() = Some(stats.clone());
+        })
+    }
+
+    fn consume(self) -> Option<ExecutionSummaryCounts> {
+        self.collected_stats.lock().unwrap().take()
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+struct SchedulerDiagnosticsCollector {
+    latest: Arc<Mutex<Option<SchedulerDiagnostics>>>,
+}
+
+impl SchedulerDiagnosticsCollector {
+    fn clear(&self) {
+        *self.latest.lock().unwrap() = None;
+    }
+
+    fn observe(&self, diagnostics: SchedulerDiagnostics) {
+        *self.latest.lock().unwrap() = Some(diagnostics);
+    }
+
+    fn snapshot(&self, io_buffer_gib: Option<u64>) -> SchedulerDiagnostics {
+        self.latest
+            .lock()
+            .unwrap()
+            .as_ref()
+            .copied()
+            .unwrap_or_else(|| diagnostics_from_scan_stats(ScanStats::default(), io_buffer_gib))
+    }
+}
+
+#[derive(Debug, Clone)]
+struct SchedulerDiagnosticsLayer {
+    collector: SchedulerDiagnosticsCollector,
+}
+
+impl SchedulerDiagnosticsLayer {
+    fn new(collector: SchedulerDiagnosticsCollector) -> Self {
+        Self { collector }
+    }
+}
+
+fn is_scheduler_state_metadata(metadata: &Metadata<'_>) -> bool {
+    // The scheduler uses `tracing::enabled!` before constructing the event;
+    // that guard registers a HINT callsite, not an EVENT callsite.
+    metadata.target() == SCHEDULER_STATE_EVENT_TARGET && *metadata.level() == tracing::Level::TRACE
+}
+
+impl<S> Layer<S> for SchedulerDiagnosticsLayer
+where
+    S: Subscriber,
+{
+    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {
+        if is_scheduler_state_metadata(metadata) {
+            Interest::always()
+        } else {
+            Interest::never()
+        }
+    }
+
+    fn enabled(&self, metadata: &Metadata<'_>, _ctx: Context<'_, S>) -> bool {
+        is_scheduler_state_metadata(metadata)
+    }
+
+    fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
+        if !is_scheduler_state_metadata(event.metadata()) {
+            return;
+        }
+        let mut visitor = SchedulerDiagnosticsVisitor::default();
+        event.record(&mut visitor);
+        if let Some(diagnostics) = visitor.into_diagnostics() {
+            self.collector.observe(diagnostics);
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+struct SchedulerDiagnosticsVisitor {
+    kind: Option<SchedulerQueueKind>,
+    scheduler_iops: Option<u64>,
+    scheduler_requests: Option<u64>,
+    scheduler_bytes_read: Option<u64>,
+    io_capacity: Option<u64>,
+    iops_available: Option<u64>,
+    active_iops: Option<u64>,
+    pending_iops: Option<u64>,
+    pending_bytes: Option<u64>,
+    bytes_available: Option<i64>,
+    bytes_reserved: Option<i64>,
+    io_buffer_size_bytes: Option<u64>,
+    priorities_in_flight: Option<u64>,
+    no_backpressure: Option<bool>,
+    head_task_bytes_present: bool,
+    head_task_bytes: Option<u64>,
+    head_task_priority_high_present: bool,
+    head_task_priority_high: Option<u64>,
+    head_task_priority_low_present: bool,
+    head_task_priority_low: Option<u64>,
+    min_in_flight_priority_high_present: bool,
+    min_in_flight_priority_high: Option<u64>,
+    min_in_flight_priority_low_present: bool,
+    min_in_flight_priority_low: Option<u64>,
+    head_task_can_deliver_present: bool,
+    head_task_can_deliver: Option<bool>,
+    head_task_priority_bypass_present: bool,
+    head_task_priority_bypass: Option<bool>,
+    head_task_blocked_by_iops_present: bool,
+    head_task_blocked_by_iops: Option<bool>,
+    head_task_blocked_by_bytes_present: bool,
+    head_task_blocked_by_bytes: Option<bool>,
+}
+
+impl SchedulerDiagnosticsVisitor {
+    fn into_diagnostics(self) -> Option<SchedulerDiagnostics> {
+        Some(SchedulerDiagnostics {
+            kind: self.kind?,
+            stats: ScanStats {
+                iops: self.scheduler_iops.unwrap_or_default(),
+                requests: self.scheduler_requests.unwrap_or_default(),
+                bytes_read: self.scheduler_bytes_read.unwrap_or_default(),
+            },
+            io_capacity: self.io_capacity.unwrap_or_default(),
+            iops_available: self.iops_available.unwrap_or_default(),
+            active_iops: self.active_iops.unwrap_or_default(),
+            pending_iops: self.pending_iops.unwrap_or_default(),
+            pending_bytes: self.pending_bytes.unwrap_or_default(),
+            bytes_available: self.bytes_available.unwrap_or_default(),
+            bytes_reserved: self.bytes_reserved.unwrap_or_default(),
+            io_buffer_size_bytes: self.io_buffer_size_bytes.unwrap_or_default(),
+            priorities_in_flight: self.priorities_in_flight.unwrap_or_default(),
+            no_backpressure: self.no_backpressure.unwrap_or(false),
+            head_task_bytes: optional_u64(self.head_task_bytes_present, self.head_task_bytes),
+            head_task_priority_high: optional_u64(
+                self.head_task_priority_high_present,
+                self.head_task_priority_high,
+            ),
+            head_task_priority_low: optional_u64(
+                self.head_task_priority_low_present,
+                self.head_task_priority_low,
+            ),
+            min_in_flight_priority_high: optional_u64(
+                self.min_in_flight_priority_high_present,
+                self.min_in_flight_priority_high,
+            ),
+            min_in_flight_priority_low: optional_u64(
+                self.min_in_flight_priority_low_present,
+                self.min_in_flight_priority_low,
+            ),
+            head_task_can_deliver: optional_bool(
+                self.head_task_can_deliver_present,
+                self.head_task_can_deliver,
+            ),
+            head_task_priority_bypass: optional_bool(
+                self.head_task_priority_bypass_present,
+                self.head_task_priority_bypass,
+            ),
+            head_task_blocked_by_iops: optional_bool(
+                self.head_task_blocked_by_iops_present,
+                self.head_task_blocked_by_iops,
+            ),
+            head_task_blocked_by_bytes: optional_bool(
+                self.head_task_blocked_by_bytes_present,
+                self.head_task_blocked_by_bytes,
+            ),
+        })
+    }
+}
+
+impl Visit for SchedulerDiagnosticsVisitor {
+    fn record_bool(&mut self, field: &Field, value: bool) {
+        match field.name() {
+            "no_backpressure" => self.no_backpressure = Some(value),
+            "head_task_bytes_present" => self.head_task_bytes_present = value,
+            "head_task_priority_high_present" => self.head_task_priority_high_present = value,
+            "head_task_priority_low_present" => self.head_task_priority_low_present = value,
+            "min_in_flight_priority_high_present" => {
+                self.min_in_flight_priority_high_present = value;
+            }
+            "min_in_flight_priority_low_present" => {
+                self.min_in_flight_priority_low_present = value;
+            }
+            "head_task_can_deliver_present" => self.head_task_can_deliver_present = value,
+            "head_task_can_deliver" => self.head_task_can_deliver = Some(value),
+            "head_task_priority_bypass_present" => {
+                self.head_task_priority_bypass_present = value;
+            }
+            "head_task_priority_bypass" => self.head_task_priority_bypass = Some(value),
+            "head_task_blocked_by_iops_present" => {
+                self.head_task_blocked_by_iops_present = value;
+            }
+            "head_task_blocked_by_iops" => self.head_task_blocked_by_iops = Some(value),
+            "head_task_blocked_by_bytes_present" => {
+                self.head_task_blocked_by_bytes_present = value;
+            }
+            "head_task_blocked_by_bytes" => self.head_task_blocked_by_bytes = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_i64(&mut self, field: &Field, value: i64) {
+        match field.name() {
+            "bytes_available" => self.bytes_available = Some(value),
+            "bytes_reserved" => self.bytes_reserved = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_u64(&mut self, field: &Field, value: u64) {
+        match field.name() {
+            "scheduler_iops" => self.scheduler_iops = Some(value),
+            "scheduler_requests" => self.scheduler_requests = Some(value),
+            "scheduler_bytes_read" => self.scheduler_bytes_read = Some(value),
+            "io_capacity" => self.io_capacity = Some(value),
+            "iops_available" => self.iops_available = Some(value),
+            "active_iops" => self.active_iops = Some(value),
+            "pending_iops" => self.pending_iops = Some(value),
+            "pending_bytes" => self.pending_bytes = Some(value),
+            "io_buffer_size_bytes" => self.io_buffer_size_bytes = Some(value),
+            "priorities_in_flight" => self.priorities_in_flight = Some(value),
+            "head_task_bytes" => self.head_task_bytes = Some(value),
+            "head_task_priority_high" => self.head_task_priority_high = Some(value),
+            "head_task_priority_low" => self.head_task_priority_low = Some(value),
+            "min_in_flight_priority_high" => self.min_in_flight_priority_high = Some(value),
+            "min_in_flight_priority_low" => self.min_in_flight_priority_low = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_str(&mut self, field: &Field, value: &str) {
+        if field.name() == "queue_kind" {
+            self.kind = match value {
+                "standard" => Some(SchedulerQueueKind::Standard),
+                "lite" => Some(SchedulerQueueKind::Lite),
+                _ => None,
+            };
+        }
+    }
+
+    fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {}
+}
+
+fn optional_u64(present: bool, value: Option<u64>) -> Option<u64> {
+    present.then(|| value.unwrap_or_default())
+}
+
+fn optional_bool(present: bool, value: Option<bool>) -> Option<bool> {
+    present.then(|| value.unwrap_or(false))
+}
+
+fn diagnostics_from_scan_stats(
+    stats: ScanStats,
+    io_buffer_gib: Option<u64>,
+) -> SchedulerDiagnostics {
+    SchedulerDiagnostics {
+        kind: SchedulerQueueKind::Standard,
+        stats,
+        io_capacity: 0,
+        iops_available: 0,
+        active_iops: 0,
+        pending_iops: 0,
+        pending_bytes: 0,
+        bytes_available: 0,
+        bytes_reserved: 0,
+        io_buffer_size_bytes: io_buffer_gib.map(|value| value * GIB).unwrap_or_default(),
+        priorities_in_flight: 0,
+        no_backpressure: false,
+        head_task_bytes: None,
+        head_task_priority_high: None,
+        head_task_priority_low: None,
+        min_in_flight_priority_high: None,
+        min_in_flight_priority_low: None,
+        head_task_can_deliver: None,
+        head_task_priority_bypass: None,
+        head_task_blocked_by_iops: None,
+        head_task_blocked_by_bytes: None,
+    }
+}
+
+fn scan_stats_from_execution_summary(summary: &ExecutionSummaryCounts) -> ScanStats {
+    ScanStats {
+        iops: summary.iops as u64,
+        requests: summary.requests as u64,
+        bytes_read: summary.bytes_read as u64,
+    }
+}
+
+fn sample_json(
+    started: Instant,
+    counters: &SharedCounters,
+    diagnostics: SchedulerDiagnostics,
+    decode_in_flight: usize,
+    channel_buffered: usize,
+    last: &mut LastSample,
+) -> Value {
+    let elapsed = started.elapsed();
+    let interval = elapsed.saturating_sub(last.elapsed);
+    let interval_secs = interval.as_secs_f64();
+    let rows = counters.rows_completed.load(Ordering::Relaxed);
+    let batches = counters.batches_completed.load(Ordering::Relaxed);
+    let arrow_bytes = counters.arrow_bytes.load(Ordering::Relaxed);
+    let scheduler_stats = diagnostics.stats;
+    let delta_scheduler_bytes =
+        diff_u64(scheduler_stats.bytes_read, last.scheduler_stats.bytes_read);
+    let delta_rows = diff_u64(rows, last.rows);
+    let delta_arrow_bytes = diff_u64(arrow_bytes, last.arrow_bytes);
+    let physical_gbps = if interval_secs > 0.0 {
+        delta_scheduler_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0
+    } else {
+        0.0
+    };
+    let logical_gbps = if interval_secs > 0.0 {
+        delta_arrow_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0
+    } else {
+        0.0
+    };
+    let rows_per_second = if interval_secs > 0.0 {
+        delta_rows as f64 / interval_secs
+    } else {
+        0.0
+    };
+
+    last.elapsed = elapsed;
+    last.scheduler_stats = scheduler_stats;
+    last.rows = rows;
+    last.batches = batches;
+    last.arrow_bytes = arrow_bytes;
+
+    json!({
+        "elapsed_seconds": elapsed.as_secs_f64(),
+        "interval_seconds": interval_secs,
+        "physical_gbps": physical_gbps,
+        "logical_gbps": logical_gbps,
+        "rows_per_second": rows_per_second,
+        "rows": rows,
+        "batches": batches,
+        "arrow_bytes": arrow_bytes,
+        "delta_rows": delta_rows,
+        "delta_arrow_bytes": delta_arrow_bytes,
+        "delta_scheduler_bytes": delta_scheduler_bytes,
+        "fragments_started": counters.fragments_started.load(Ordering::Relaxed),
+        "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed),
+        "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed),
+        "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed),
+        "batches_completed": counters.batches_completed.load(Ordering::Relaxed),
+        "decode_in_flight": decode_in_flight,
+        "channel_buffered": channel_buffered,
+        "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)),
+        "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)),
+        "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)),
+        "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)),
+        "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)),
+        "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)),
+        "scheduler": diagnostics_json(diagnostics),
+    })
+}
+
+async fn run_scanner_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+
+    let mut remaining_rows = config.limit_rows;
+    let mut planned_fragments = 0usize;
+    let mut planned_rows = 0u64;
+    for fragment in dataset.fragments().iter() {
+        if remaining_rows == 0 {
+            break;
+        }
+        let fragment_rows = fragment
+            .num_rows()
+            .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))?
+            as u64;
+        let rows = fragment_rows.min(remaining_rows);
+        planned_fragments += 1;
+        planned_rows += rows;
+        remaining_rows -= rows;
+    }
+    if planned_fragments == 0 {
+        return Err("no fragments selected".into());
+    }
+
+    let counters = Arc::new(SharedCounters::default());
+    let stats_holder = ExecutionStatsHolder::default();
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+
+    let mut scanner = dataset.scan();
+    if let Some(columns) = config.columns.as_ref() {
+        scanner.project(columns)?;
+    }
+    scanner
+        .batch_size(config.batch_size as usize)
+        .scan_in_order(false)
+        .scan_stats_callback(stats_holder.get_setter());
+    if config.batch_concurrency > 0 {
+        scanner
+            .batch_readahead(config.batch_concurrency)
+            .target_parallelism(config.batch_concurrency);
+    }
+    if config.fragment_concurrency > 0 {
+        scanner.fragment_readahead(config.fragment_concurrency);
+    }
+    if let Some(file_reader_options) = file_reader_options(config) {
+        scanner.with_file_reader_options(file_reader_options);
+    }
+    if let Some(batch_size_bytes) = config.batch_size_bytes {
+        scanner.batch_size_bytes(batch_size_bytes);
+    }
+    if let Some(io_buffer_gib) = io_buffer_gib {
+        scanner.io_buffer_size(io_buffer_gib * GIB);
+    }
+    let limit_rows = i64::try_from(config.limit_rows)
+        .map_err(|_| "--limit-rows is too large for scanner limit")?;
+    scanner.limit(Some(limit_rows), None)?;
+
+    let mut stream = scanner.try_into_stream().await?;
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+    loop {
+        tokio::select! {
+            maybe_batch = stream.next() => {
+                let Some(batch) = maybe_batch else {
+                    break;
+                };
+                let batch = batch?;
+                let batch_bytes = if config.skip_batch_byte_accounting {
+                    0
+                } else {
+                    batch.get_array_memory_size() as u64
+                };
+                rows += batch.num_rows() as u64;
+                batches += 1;
+                arrow_bytes += batch_bytes;
+                counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                counters
+                    .rows_completed
+                    .fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+                counters.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed);
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    0,
+                    0,
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    drop(stream);
+
+    let elapsed = started.elapsed();
+    let summary = stats_holder
+        .consume()
+        .ok_or("scanner execution stats callback did not run")?;
+    let scheduler_stats = scan_stats_from_execution_summary(&summary);
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler_stats;
+    let cpu_after = read_cpu_sample();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        0,
+        0,
+        &mut last_sample,
+    ));
+
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments,
+        planned_rows,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: 0,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+async fn run_dataset_take_case(
+    config: &Config,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = DatasetBuilder::from_uri(&config.uri)
+        .with_version(config.dataset_version)
+        .load()
+        .await?;
+    let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?);
+    let total_rows = dataset
+        .fragments()
+        .iter()
+        .map(|fragment| {
+            fragment
+                .num_rows()
+                .map(|rows| rows as u64)
+                .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))
+        })
+        .collect::<std::result::Result<Vec<_>, _>>()?
+        .into_iter()
+        .sum::<u64>();
+    if total_rows == 0 {
+        return Err("dataset has no rows".into());
+    }
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    const STRIDE: u64 = 104_729;
+
+    for repetition in 0..config.take_repetitions {
+        let row_ids = (0..config.limit_rows)
+            .map(|offset| {
+                repetition
+                    .wrapping_mul(STRIDE)
+                    .wrapping_add(offset.wrapping_mul(STRIDE))
+                    % total_rows
+            })
+            .collect::<Vec<_>>();
+        let batch = dataset
+            .take(&row_ids, ProjectionRequest::Schema(projection.clone()))
+            .await?;
+        if batch.num_rows() as u64 != config.limit_rows {
+            return Err(format!(
+                "take_rows returned {} rows, expected {}",
+                batch.num_rows(),
+                config.limit_rows
+            )
+            .into());
+        }
+        black_box(&batch);
+        rows += batch.num_rows() as u64;
+        batches += 1;
+        let batch_bytes = if config.skip_batch_byte_accounting {
+            0
+        } else {
+            batch.get_array_memory_size() as u64
+        };
+        arrow_bytes += batch_bytes;
+        counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+        counters
+            .rows_completed
+            .fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+        counters
+            .arrow_bytes
+            .fetch_add(batch_bytes, Ordering::Relaxed);
+    }
+
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments: dataset.fragments().len(),
+        planned_rows: total_rows,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: 0,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: scheduler_diagnostics.snapshot(None),
+        counters,
+        samples: Vec::new(),
+    })
+}
+
+enum RawInFlight {
+    Unordered(FuturesUnordered<BoxFuture<'static, lance_core::Result<usize>>>),
+    Ordered(FuturesOrdered<BoxFuture<'static, lance_core::Result<usize>>>),
+}
+
+impl RawInFlight {
+    fn new(mode: RawCompletionMode) -> Self {
+        match mode {
+            RawCompletionMode::Unordered => Self::Unordered(FuturesUnordered::new()),
+            RawCompletionMode::Ordered => Self::Ordered(FuturesOrdered::new()),
+        }
+    }
+
+    fn len(&self) -> usize {
+        match self {
+            Self::Unordered(in_flight) => in_flight.len(),
+            Self::Ordered(in_flight) => in_flight.len(),
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        match self {
+            Self::Unordered(in_flight) => in_flight.is_empty(),
+            Self::Ordered(in_flight) => in_flight.is_empty(),
+        }
+    }
+
+    fn push(&mut self, future: BoxFuture<'static, lance_core::Result<usize>>) {
+        match self {
+            Self::Unordered(in_flight) => in_flight.push(future),
+            Self::Ordered(in_flight) => in_flight.push_back(future),
+        }
+    }
+
+    async fn next(&mut self) -> Option<lance_core::Result<usize>> {
+        match self {
+            Self::Unordered(in_flight) => in_flight.next().await,
+            Self::Ordered(in_flight) => in_flight.next().await,
+        }
+    }
+}
+
+fn raw_read_future(
+    file_scheduler: FileScheduler,
+    range: Range<u64>,
+    priority: u64,
+    raw_submit_mode: RawSubmitMode,
+    read_chunk_size: u64,
+    counters: Arc<SharedCounters>,
+) -> BoxFuture<'static, lance_core::Result<usize>> {
+    async move {
+        match raw_submit_mode {
+            RawSubmitMode::Single => {
+                let bytes = file_scheduler.submit_single(range, priority).await?;
+                Ok(bytes.len())
+            }
+            RawSubmitMode::SplitNoConcat => {
+                let ranges = split_range_by_size(range, read_chunk_size);
+                let bytes = file_scheduler.submit_request(ranges, priority).await?;
+                Ok(bytes.iter().map(bytes::Bytes::len).sum())
+            }
+            RawSubmitMode::SplitConcat => {
+                let ranges = split_range_by_size(range, read_chunk_size);
+                let bytes = file_scheduler.submit_request(ranges, priority).await?;
+                let reassemble_started = Instant::now();
+                let total_size = bytes.iter().map(bytes::Bytes::len).sum();
+                let mut combined = Vec::with_capacity(total_size);
+                for chunk in bytes {
+                    combined.extend_from_slice(&chunk);
+                }
+                add_duration(&counters.raw_reassemble_ns, reassemble_started.elapsed());
+                let len = combined.len();
+                black_box(&combined);
+                Ok(len)
+            }
+        }
+    }
+    .boxed()
+}
+
+fn split_range_by_size(range: Range<u64>, chunk_size: u64) -> Vec<Range<u64>> {
+    let range_size = range.end - range.start;
+    if range_size <= chunk_size {
+        return vec![range];
+    }
+
+    let num_chunks = range_size.div_ceil(chunk_size);
+    let per_chunk = range_size / num_chunks;
+    let mut ranges = Vec::with_capacity(num_chunks as usize);
+    for idx in 0..num_chunks {
+        let start = range.start + idx * per_chunk;
+        let end = if idx == num_chunks - 1 {
+            range.end
+        } else {
+            start + per_chunk
+        };
+        ranges.push(start..end);
+    }
+    ranges
+}
+
+fn push_split_planned_ranges(
+    planned: &mut Vec<(usize, Range<u64>)>,
+    file_idx: usize,
+    range: Range<u64>,
+    chunk_size: u64,
+    remaining: &mut u64,
+) {
+    let mut start = range.start;
+    while start < range.end && *remaining > 0 {
+        let bytes_to_read = chunk_size.min(range.end - start).min(*remaining);
+        if bytes_to_read == 0 {
+            break;
+        }
+        let end = start + bytes_to_read;
+        planned.push((file_idx, start..end));
+        *remaining -= bytes_to_read;
+        start = end;
+    }
+}
+
+fn push_split_ranges(ranges: &mut Vec<Range<u64>>, range: Range<u64>, chunk_size: u64) {
+    let mut start = range.start;
+    while start < range.end {
+        let bytes_to_read = chunk_size.min(range.end - start);
+        if bytes_to_read == 0 {
+            break;
+        }
+        let end = start + bytes_to_read;
+        ranges.push(start..end);
+        start = end;
+    }
+}
+
+async fn run_scheduler_raw_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let target_bytes = config
+        .target_bytes
+        .ok_or("--target-bytes is required for --backend scheduler-raw")?;
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler_config = io_buffer_gib
+        .map(|gib| SchedulerConfig::new(gib * GIB))
+        .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref()));
+    let scheduler = ScanScheduler::new(object_store, scheduler_config);
+
+    let (selected_files, planned) = match config.raw_range_mode {
+        RawRangeMode::FileSequential => {
+            let mut selected_files = Vec::new();
+            let mut selected_file_bytes = 0u64;
+            for fragment in dataset.fragments().iter() {
+                for data_file in &fragment.files {
+                    if data_file.base_id.is_some() {
+                        continue;
+                    }
+                    let Some(file_size) = data_file.file_size_bytes.get() else {
+                        continue;
+                    };
+                    let path = dataset.data_dir().join(data_file.path.as_str());
+                    let file_scheduler = scheduler
+                        .open_file_with_priority(&path, 0, &data_file.file_size_bytes)
+                        .await?;
+                    selected_file_bytes += file_size.get();
+                    selected_files.push((file_scheduler, file_size.get()));
+                    if selected_file_bytes >= target_bytes {
+                        break;
+                    }
+                }
+                if selected_file_bytes >= target_bytes {
+                    break;
+                }
+            }
+            if selected_files.is_empty() {
+                return Err("scheduler-raw found no data files with known sizes".into());
+            }
+
+            let mut offsets = vec![0u64; selected_files.len()];
+            let mut planned = Vec::new();
+            let mut remaining = target_bytes;
+            let mut file_idx = 0usize;
+            while remaining > 0 {
+                let idx = file_idx % selected_files.len();
+                let file_size = selected_files[idx].1;
+                if offsets[idx] >= file_size {
+                    offsets[idx] = 0;
+                }
+                let available = file_size - offsets[idx];
+                let bytes_to_read = config.raw_range_size_bytes.min(available).min(remaining);
+                let start = offsets[idx];
+                let end = start + bytes_to_read;
+                planned.push((idx, start..end));
+                offsets[idx] = end;
+                remaining -= bytes_to_read;
+                file_idx += 1;
+            }
+            (selected_files, planned)
+        }
+        RawRangeMode::MetadataPages | RawRangeMode::MetadataPagesRoundRobin => {
+            let mut selected_files = Vec::new();
+            let mut per_file_ranges = Vec::<Vec<Range<u64>>>::new();
+            let mut candidate_bytes = 0u64;
+
+            'fragments: for fragment in dataset.fragments().iter() {
+                for data_file in &fragment.files {
+                    if data_file.base_id.is_some() {
+                        continue;
+                    }
+                    if data_file.file_size_bytes.get().is_none() {
+                        continue;
+                    }
+                    let path = dataset.data_dir().join(data_file.path.as_str());
+                    let file_scheduler = scheduler
+                        .open_file_with_priority(&path, 0, &data_file.file_size_bytes)
+                        .await?;
+                    let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?;
+                    let mut file_ranges = Vec::new();
+
+                    let raw_column_indices = config
+                        .raw_column_indices
+                        .clone()
+                        .unwrap_or_else(|| (0..metadata.column_infos.len() as u32).collect());
+                    for column_index in raw_column_indices {
+                        let column_info = metadata
+                            .column_infos
+                            .get(column_index as usize)
+                            .ok_or_else(|| {
+                                format!(
+                                    "raw metadata-pages requested column index {column_index} but file has {} columns",
+                                    metadata.column_infos.len()
+                                )
+                            })?;
+                        for page in column_info.page_infos.iter() {
+                            for (offset, size) in page.buffer_offsets_and_sizes.iter() {
+                                if *size == 0 {
+                                    continue;
+                                }
+                                push_split_ranges(
+                                    &mut file_ranges,
+                                    *offset..(*offset + *size),
+                                    config.raw_range_size_bytes,
+                                );
+                            }
+                        }
+                    }
+
+                    if !file_ranges.is_empty() {
+                        candidate_bytes += file_ranges
+                            .iter()
+                            .map(|range| range.end - range.start)
+                            .sum::<u64>();
+                        selected_files.push((
+                            file_scheduler,
+                            data_file.file_size_bytes.get().unwrap().get(),
+                        ));
+                        per_file_ranges.push(file_ranges);
+                        if candidate_bytes >= target_bytes {
+                            break 'fragments;
+                        }
+                    }
+                }
+            }
+            if selected_files.is_empty() || per_file_ranges.is_empty() {
+                return Err("scheduler-raw metadata-pages found no readable page buffers".into());
+            }
+
+            let mut planned = Vec::new();
+            let mut remaining = target_bytes;
+            match config.raw_range_mode {
+                RawRangeMode::MetadataPages => {
+                    'ranges: for (file_idx, ranges) in per_file_ranges.iter().enumerate() {
+                        for range in ranges {
+                            push_split_planned_ranges(
+                                &mut planned,
+                                file_idx,
+                                range.clone(),
+                                config.raw_range_size_bytes,
+                                &mut remaining,
+                            );
+                            if remaining == 0 {
+                                break 'ranges;
+                            }
+                        }
+                    }
+                }
+                RawRangeMode::MetadataPagesRoundRobin => {
+                    let mut positions = vec![0usize; per_file_ranges.len()];
+                    while remaining > 0 {
+                        let mut made_progress = false;
+                        for (file_idx, ranges) in per_file_ranges.iter().enumerate() {
+                            if positions[file_idx] >= ranges.len() {
+                                continue;
+                            }
+                            let range = ranges[positions[file_idx]].clone();
+                            positions[file_idx] += 1;
+                            made_progress = true;
+                            push_split_planned_ranges(
+                                &mut planned,
+                                file_idx,
+                                range,
+                                config.raw_range_size_bytes,
+                                &mut remaining,
+                            );
+                            if remaining == 0 {
+                                break;
+                            }
+                        }
+                        if !made_progress {
+                            break;
+                        }
+                    }
+                }
+                RawRangeMode::FileSequential => unreachable!(),
+            }
+
+            if remaining > 0 {
+                return Err(format!(
+                    "scheduler-raw metadata-pages planned {} bytes but target is {target_bytes}",
+                    target_bytes - remaining
+                )
+                .into());
+            }
+            (selected_files, planned)
+        }
+    };
+    let planned_bytes = planned
+        .iter()
+        .map(|(_, range)| range.end - range.start)
+        .sum::<u64>();
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+    let mut in_flight = RawInFlight::new(config.raw_completion_mode);
+    let mut next_range = 0usize;
+    let read_chunk_size = config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE);
+    while next_range < planned.len() && in_flight.len() < config.batch_concurrency {
+        let (idx, range) = planned[next_range].clone();
+        in_flight.push(raw_read_future(
+            selected_files[idx].0.clone(),
+            range,
+            next_range as u64,
+            config.raw_submit_mode,
+            read_chunk_size,
+            counters.clone(),
+        ));
+        next_range += 1;
+    }
+
+    let mut bytes_read = 0u64;
+    let mut requests_completed = 0u64;
+    while !in_flight.is_empty() {
+        tokio::select! {
+            maybe_bytes = in_flight.next() => {
+                let bytes = maybe_bytes.expect("raw read future disappeared")?;
+                let bytes = bytes as u64;
+                bytes_read += bytes;
+                requests_completed += 1;
+                counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                counters.arrow_bytes.fetch_add(bytes, Ordering::Relaxed);
+                if next_range < planned.len() {
+                    let (idx, range) = planned[next_range].clone();
+                    in_flight.push(raw_read_future(
+                        selected_files[idx].0.clone(),
+                        range,
+                        next_range as u64,
+                        config.raw_submit_mode,
+                        read_chunk_size,
+                        counters.clone(),
+                    ));
+                    next_range += 1;
+                }
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    in_flight.len(),
+                    planned.len().saturating_sub(next_range),
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    counters.arrow_bytes.store(bytes_read, Ordering::Relaxed);
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler.stats();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        in_flight.len(),
+        0,
+        &mut last_sample,
+    ));
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+
+    Ok(CaseStats {
+        rows: 0,
+        batches: requests_completed,
+        arrow_bytes: bytes_read,
+        planned_fragments: selected_files.len(),
+        planned_rows: planned_bytes,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: config.batch_concurrency,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+async fn run_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?);
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler_config = io_buffer_gib
+        .map(|gib| SchedulerConfig::new(gib * GIB))
+        .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref()));
+    let scheduler = ScanScheduler::new(object_store, scheduler_config);
+
+    let mut planned = Vec::new();
+    let mut remaining_rows = config.limit_rows;
+    for fragment in dataset.fragments().iter() {
+        if remaining_rows == 0 {
+            break;
+        }
+        let fragment_rows = fragment
+            .num_rows()
+            .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))?
+            as u64;
+        let rows = fragment_rows.min(remaining_rows);
+        planned.push((fragment.clone(), rows));
+        remaining_rows -= rows;
+    }
+    if planned.is_empty() {
+        return Err("no fragments selected".into());
+    }
+    let planned_rows: u64 = planned.iter().map(|(_, rows)| *rows).sum();
+    let planned_fragments = planned.len();
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+
+    let (tx, mut rx) = tokio::sync::mpsc::channel::<
+        BoxFuture<'static, lance_core::Result<RecordBatch>>,
+    >(config.batch_concurrency * 2);
+    let producer = if config.detach_fragment_streams {
+        tokio::spawn({
+            let dataset = dataset.clone();
+            let projection = projection.clone();
+            let scheduler = scheduler.clone();
+            let counters = counters.clone();
+            let batch_size = config.batch_size;
+            let file_reader_options = file_reader_options(config);
+            let fragment_concurrency = config.fragment_concurrency;
+            async move {
+                let drainers = futures::stream::iter(planned.into_iter().enumerate())
+                    .map({
+                        move |(priority, (fragment, rows))| {
+                            let dataset = dataset.clone();
+                            let projection = projection.clone();
+                            let scheduler = scheduler.clone();
+                            let tx = tx.clone();
+                            let counters = counters.clone();
+                            let file_reader_options = file_reader_options.clone();
+                            async move {
+                                counters.fragments_started.fetch_add(1, Ordering::Relaxed);
+                                let file_fragment = FileFragment::new(dataset, fragment);
+                                let read_config = FragReadConfig::default()
+                                    .with_scan_scheduler(scheduler)
+                                    .with_reader_priority(priority as u32);
+                                let read_config = if let Some(file_reader_options) =
+                                    file_reader_options.clone()
+                                {
+                                    read_config.with_file_reader_options(file_reader_options)
+                                } else {
+                                    read_config
+                                };
+
+                                let open_started = Instant::now();
+                                let reader =
+                                    file_fragment.open(projection.as_ref(), read_config).await?;
+                                add_duration(&counters.open_reader_ns, open_started.elapsed());
+
+                                let create_stream_started = Instant::now();
+                                let mut read_stream =
+                                    reader.read_ranges(vec![0..rows].into(), batch_size).await?;
+                                add_duration(
+                                    &counters.read_stream_create_ns,
+                                    create_stream_started.elapsed(),
+                                );
+
+                                let drainer = tokio::spawn(async move {
+                                    loop {
+                                        let next_started = Instant::now();
+                                        let maybe_batch_fut = read_stream.next().await;
+                                        add_duration(
+                                            &counters.next_batch_poll_ns,
+                                            next_started.elapsed(),
+                                        );
+                                        let Some(batch_fut) = maybe_batch_fut else {
+                                            break;
+                                        };
+                                        counters
+                                            .batch_futures_emitted
+                                            .fetch_add(1, Ordering::Relaxed);
+                                        let send_started = Instant::now();
+                                        tx.send(batch_fut)
+                                            .await
+                                            .map_err(|_| "batch consumer dropped")?;
+                                        add_duration(
+                                            &counters.channel_send_wait_ns,
+                                            send_started.elapsed(),
+                                        );
+                                    }
+                                    counters.fragments_completed.fetch_add(1, Ordering::Relaxed);
+                                    Ok::<_, Error>(())
+                                });
+                                Ok::<_, Error>(drainer)
+                            }
+                        }
+                    })
+                    .buffer_unordered(fragment_concurrency)
+                    .try_collect::<Vec<_>>()
+                    .await?;
+
+                for drainer in drainers {
+                    drainer.await.map_err(Error::from)??;
+                }
+                Ok::<_, Error>(())
+            }
+        })
+    } else {
+        tokio::spawn({
+            let dataset = dataset.clone();
+            let projection = projection.clone();
+            let scheduler = scheduler.clone();
+            let counters = counters.clone();
+            let batch_size = config.batch_size;
+            let file_reader_options = file_reader_options(config);
+            let fragment_concurrency = config.fragment_concurrency;
+            async move {
+                futures::stream::iter(planned.into_iter().enumerate())
+                    .map({
+                        move |(priority, (fragment, rows))| {
+                            let dataset = dataset.clone();
+                            let projection = projection.clone();
+                            let scheduler = scheduler.clone();
+                            let tx = tx.clone();
+                            let counters = counters.clone();
+                            let file_reader_options = file_reader_options.clone();
+                            async move {
+                                counters.fragments_started.fetch_add(1, Ordering::Relaxed);
+                                let file_fragment = FileFragment::new(dataset, fragment);
+                                let read_config = FragReadConfig::default()
+                                    .with_scan_scheduler(scheduler)
+                                    .with_reader_priority(priority as u32);
+                                let read_config = if let Some(file_reader_options) =
+                                    file_reader_options.clone()
+                                {
+                                    read_config.with_file_reader_options(file_reader_options)
+                                } else {
+                                    read_config
+                                };
+
+                                let open_started = Instant::now();
+                                let reader =
+                                    file_fragment.open(projection.as_ref(), read_config).await?;
+                                add_duration(&counters.open_reader_ns, open_started.elapsed());
+
+                                let create_stream_started = Instant::now();
+                                let mut read_stream =
+                                    reader.read_ranges(vec![0..rows].into(), batch_size).await?;
+                                add_duration(
+                                    &counters.read_stream_create_ns,
+                                    create_stream_started.elapsed(),
+                                );
+
+                                loop {
+                                    let next_started = Instant::now();
+                                    let maybe_batch_fut = read_stream.next().await;
+                                    add_duration(
+                                        &counters.next_batch_poll_ns,
+                                        next_started.elapsed(),
+                                    );
+                                    let Some(batch_fut) = maybe_batch_fut else {
+                                        break;
+                                    };
+                                    counters
+                                        .batch_futures_emitted
+                                        .fetch_add(1, Ordering::Relaxed);
+                                    let send_started = Instant::now();
+                                    tx.send(batch_fut)
+                                        .await
+                                        .map_err(|_| "batch consumer dropped")?;
+                                    add_duration(
+                                        &counters.channel_send_wait_ns,
+                                        send_started.elapsed(),
+                                    );
+                                }
+                                counters.fragments_completed.fetch_add(1, Ordering::Relaxed);
+                                Ok::<_, Error>(())
+                            }
+                        }
+                    })
+                    .buffer_unordered(fragment_concurrency)
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                Ok::<_, Error>(())
+            }
+        })
+    };
+
+    let mut in_flight = FuturesUnordered::new();
+    let skip_batch_byte_accounting = config.skip_batch_byte_accounting;
+    let drop_read_tasks = config.drop_read_tasks;
+    let mut producer_done = false;
+    let mut producer_finished_at = None;
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    let mut peak_decode_in_flight = 0usize;
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+
+    loop {
+        if producer_done && in_flight.is_empty() {
+            break;
+        }
+        tokio::select! {
+            maybe_batch_fut = rx.recv(), if !producer_done && in_flight.len() < config.batch_concurrency => {
+                if let Some(batch_fut) = maybe_batch_fut {
+                    counters.batch_futures_received.fetch_add(1, Ordering::Relaxed);
+                    if drop_read_tasks {
+                        drop(batch_fut);
+                        counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                        batches += 1;
+                        continue;
+                    }
+                    let counters_for_task = counters.clone();
+                    in_flight.push(async move {
+                        let decode_started = Instant::now();
+                        let batch = batch_fut.await?;
+                        let batch_bytes = if skip_batch_byte_accounting {
+                            0
+                        } else {
+                            batch.get_array_memory_size() as u64
+                        };
+                        add_duration(&counters_for_task.decode_ns, decode_started.elapsed());
+                        counters_for_task.batches_completed.fetch_add(1, Ordering::Relaxed);
+                        counters_for_task.rows_completed.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+                        counters_for_task.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed);
+                        Ok::<_, lance_core::Error>((batch, batch_bytes))
+                    });
+                    peak_decode_in_flight = peak_decode_in_flight.max(in_flight.len());
+                } else {
+                    producer_done = true;
+                    producer_finished_at = Some(started.elapsed());
+                }
+            }
+            maybe_batch = in_flight.next(), if !in_flight.is_empty() => {
+                let (batch, batch_bytes) = maybe_batch.expect("in-flight batch future disappeared")?;
+                rows += batch.num_rows() as u64;
+                batches += 1;
+                arrow_bytes += batch_bytes;
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    in_flight.len(),
+                    rx.len(),
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    producer.await??;
+
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler.stats();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        in_flight.len(),
+        rx.len(),
+        &mut last_sample,
+    ));
+
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments,
+        planned_rows,
+        elapsed,
+        producer_finished_at,
+        peak_decode_in_flight,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+fn read_cpu_sample() -> Option<CpuSample> {
+    let contents = fs::read_to_string("/proc/stat").ok()?;
+    let line = contents.lines().next()?;
+    let values = line
+        .split_whitespace()
+        .skip(1)
+        .map(|value| value.parse::<u64>())
+        .collect::<std::result::Result<Vec<_>, _>>()
+        .ok()?;
+    if values.len() < 5 {
+        return None;
+    }
+
+    let idle = values[3] + values[4];
+    let total = values.iter().sum();
+    Some(CpuSample { idle, total })
+}
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or_default()
+}
+
+fn current_commit() -> String {
+    option_env!("LANCE_BENCH_COMMIT")
+        .or_else(|| option_env!("GIT_COMMIT"))
+        .unwrap_or("unknown")
+        .to_string()
+}
+
+fn env_var(name: &str) -> Option<String> {
+    env::var(name).ok()
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let config = parse_args()?;
+    if config.describe_layout {
+        return describe_layout(&config).await;
+    }
+    let scheduler_diagnostics = SchedulerDiagnosticsCollector::default();
+    let subscriber = tracing_subscriber::registry().with(SchedulerDiagnosticsLayer::new(
+        scheduler_diagnostics.clone(),
+    ));
+    tracing::subscriber::set_global_default(subscriber).map_err(|error| {
+        std::io::Error::other(format!(
+            "failed to install scheduler diagnostics subscriber: {error}"
+        ))
+    })?;
+
+    fs::create_dir_all(&config.out_dir)?;
+    let output_path = format!(
+        "{}/s3_file_reader_diagnostics_{}.jsonl",
+        config.out_dir,
+        now_unix_secs()
+    );
+    let mut jsonl = String::new();
+    let commit = current_commit();
+    let instance = env_var("EC2_INSTANCE_TYPE").unwrap_or_else(|| "unknown".to_string());
+    let region = env_var("AWS_REGION")
+        .or_else(|| env_var("AWS_DEFAULT_REGION"))
+        .unwrap_or_else(|| "unknown".to_string());
+    let projection = projection_name(&config.columns);
+
+    for io_buffer_gib in &config.io_buffer_gib {
+        println!(
+            "running case={} backend={} projection={} limit_rows={} io_buffer_gib={} batch_size={} fragment_concurrency={} batch_concurrency={} sample_ms={}",
+            config.case_name,
+            config.backend.name(),
+            projection,
+            config.limit_rows,
+            io_buffer_gib
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "auto".to_string()),
+            config.batch_size,
+            config.fragment_concurrency,
+            config.batch_concurrency,
+            config.sample_ms
+        );
+        let stats = match config.backend {
+            Backend::FileReader => {
+                run_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::Scanner => {
+                run_scanner_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::SchedulerRaw => {
+                run_scheduler_raw_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::DatasetTake => run_dataset_take_case(&config, &scheduler_diagnostics).await?,
+        };
+        let elapsed_secs = stats.elapsed.as_secs_f64();
+        let scheduler_stats = stats.scheduler_diagnostics.stats;
+        let logical_gbps = if elapsed_secs > 0.0 {
+            stats.arrow_bytes as f64 * 8.0 / elapsed_secs / 1_000_000_000.0
+        } else {
+            0.0
+        };
+        let physical_gbps = if elapsed_secs > 0.0 {
+            scheduler_stats.bytes_read as f64 * 8.0 / elapsed_secs / 1_000_000_000.0
+        } else {
+            0.0
+        };
+        let rows_per_second = if elapsed_secs > 0.0 {
+            stats.rows as f64 / elapsed_secs
+        } else {
+            0.0
+        };
+        let bytes_per_row = stats.arrow_bytes.checked_div(stats.rows).unwrap_or(0);
+        let avg_bytes_per_scheduler_request = scheduler_stats
+            .bytes_read
+            .checked_div(scheduler_stats.requests)
+            .unwrap_or(0);
+        let avg_bytes_per_scheduler_iop = scheduler_stats
+            .bytes_read
+            .checked_div(scheduler_stats.iops)
+            .unwrap_or(0);
+        let counters = stats.counters.as_ref();
+        let record = json!({
+            "case": config.case_name,
+            "instance": instance,
+            "region": region,
+            "layer": config.backend.layer(),
+            "backend": config.backend.name(),
+            "dataset_uri": config.uri,
+            "dataset_version": config.dataset_version,
+            "lance_commit": commit,
+            "projection": projection,
+            "limit_rows": config.limit_rows,
+            "target_bytes": config.target_bytes,
+            "raw_range_size_bytes": config.raw_range_size_bytes,
+            "raw_range_mode": config.raw_range_mode.name(),
+            "raw_column_indices": config.raw_column_indices.clone(),
+            "raw_submit_mode": config.raw_submit_mode.name(),
+            "raw_completion_mode": config.raw_completion_mode.name(),
+            "take_repetitions": config.take_repetitions,
+            "raw_read_chunk_size_bytes": config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE),
+            "planned_rows": stats.planned_rows,
+            "planned_fragments": stats.planned_fragments,
+            "rows": stats.rows,
+            "batches": stats.batches,
+            "batch_size": config.batch_size,
+            "batch_size_bytes": config.batch_size_bytes,
+            "skip_batch_byte_accounting": config.skip_batch_byte_accounting,
+            "read_chunk_size": config.read_chunk_size,
+            "fragment_concurrency": config.fragment_concurrency,
+            "batch_concurrency": config.batch_concurrency,
+            "detach_fragment_streams": config.detach_fragment_streams,
+            "drop_read_tasks": config.drop_read_tasks,
+            "sample_ms": config.sample_ms,
+            "io_buffer_bytes": io_buffer_gib.map(|value| value * GIB),
+            "io_buffer_mode": if io_buffer_gib.is_some() { "explicit" } else { "auto" },
+            "lance_io_threads": env_var("LANCE_IO_THREADS").or_else(|| env_var("IO_THREADS")),
+            "lance_default_io_buffer_size": env_var("LANCE_DEFAULT_IO_BUFFER_SIZE"),
+            "lance_max_iop_size": env_var("LANCE_MAX_IOP_SIZE"),
+            "lance_use_lite_scheduler": env_var("LANCE_USE_LITE_SCHEDULER"),
+            "lance_inline_scheduling_threshold": env_var("LANCE_INLINE_SCHEDULING_THRESHOLD"),
+            "elapsed_seconds": elapsed_secs,
+            "producer_finished_seconds": stats.producer_finished_at.map(|duration| duration.as_secs_f64()),
+            "logical_gbps": logical_gbps,
+            "physical_gbps": physical_gbps,
+            "rows_per_second": rows_per_second,
+            "arrow_bytes": stats.arrow_bytes,
+            "bytes_per_row": bytes_per_row,
+            "avg_bytes_per_scheduler_request": avg_bytes_per_scheduler_request,
+            "avg_bytes_per_scheduler_iop": avg_bytes_per_scheduler_iop,
+            "scheduler_iops": scheduler_stats.iops,
+            "scheduler_requests": scheduler_stats.requests,
+            "scheduler_bytes_read": scheduler_stats.bytes_read,
+            "scheduler_diagnostics": diagnostics_json(stats.scheduler_diagnostics),
+            "fragments_started": counters.fragments_started.load(Ordering::Relaxed),
+            "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed),
+            "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed),
+            "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed),
+            "batches_completed": counters.batches_completed.load(Ordering::Relaxed),
+            "peak_decode_in_flight": stats.peak_decode_in_flight,
+            "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)),
+            "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)),
+            "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)),
+            "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)),
+            "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)),
+            "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)),
+            "cpu_avg": stats.cpu_avg,
+            "samples": stats.samples,
+        });
+        println!(
+            "case={} backend={} projection={} io_buffer_gib={} elapsed={:.3}s logical_gbps={:.2} physical_gbps={:.2} rows={} batches={} scheduler_bytes_read={} scheduler_iops={} scheduler_requests={} active_iops={} pending_iops={} cpu_avg={}",
+            config.case_name,
+            config.backend.name(),
+            projection,
+            io_buffer_gib
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "auto".to_string()),
+            elapsed_secs,
+            logical_gbps,
+            physical_gbps,
+            stats.rows,
+            stats.batches,
+            scheduler_stats.bytes_read,
+            scheduler_stats.iops,
+            scheduler_stats.requests,
+            stats.scheduler_diagnostics.active_iops,
+            stats.scheduler_diagnostics.pending_iops,
+            stats
+                .cpu_avg
+                .map(|value| format!("{value:.1}%"))
+                .unwrap_or_else(|| "unknown".to_string())
+        );
+        jsonl.push_str(&serde_json::to_string(&record)?);
+        jsonl.push('\n');
+        fs::write(&output_path, &jsonl)?;
+    }
+
+    println!("wrote {output_path}");
+    Ok(())
+}
diff --git a/rust/lance/src/blob.rs b/rust/lance/src/blob.rs
index b112f011419..b48a5603be6 100644
--- a/rust/lance/src/blob.rs
+++ b/rust/lance/src/blob.rs
@@ -736,12 +736,28 @@ fn validate_blob_descriptor(value: &BlobDescriptor) -> Result<()> {
     }
 }
 
+fn packed_descriptor(blob_id: u32, offset: u64, size: u64) -> Result<(BlobDescriptor, u64)> {
+    let next_offset = offset.checked_add(size).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "Packed blob writer offset overflowed: offset={offset}, size={size}"
+        ))
+    })?;
+    Ok((
+        BlobDescriptor::Packed {
+            blob_id,
+            offset,
+            size,
+        },
+        next_offset,
+    ))
+}
+
 /// Writes a Lance-owned packed sidecar blob for one data file and returns descriptors.
 pub struct PackedBlobWriter {
     object_store: ObjectStore,
     path: Path,
     blob_id: u32,
-    writer: Box<dyn Writer>,
+    writer: Option<Box<dyn Writer>>,
     offset: u64,
     values: Vec<BlobDescriptor>,
 }
@@ -759,7 +775,7 @@ impl PackedBlobWriter {
             object_store,
             path,
             blob_id,
-            writer,
+            writer: Some(writer),
             offset: 0,
             values: Vec::new(),
         })
@@ -782,10 +798,60 @@ impl PackedBlobWriter {
         Ok(())
     }
 
+    /// Append multiple logical blobs, one per iterator item.
+    ///
+    /// Each `Some(bytes)` is appended to the sidecar and records a packed
+    /// descriptor; an empty slice records a valid zero-length blob. Each `None`
+    /// records a [`BlobDescriptor::Null`] without writing any bytes, so the
+    /// descriptors returned by [`Self::finish`] stay row-aligned with the input.
+    ///
+    /// If writing fails or the future is cancelled after a partial write, no
+    /// descriptors from this call are recorded, the active writer is dropped,
+    /// and this instance cannot be reused.
+    ///
+    /// ```
+    /// # use lance::{PackedBlobWriter, Result};
+    /// # async fn write(mut writer: PackedBlobWriter) -> Result<()> {
+    /// writer
+    ///     .write_packed_blobs([Some(b"first".as_slice()), None, Some(b"second".as_slice())])
+    ///     .await?;
+    /// let descriptors = writer.finish().await?;
+    /// assert_eq!(descriptors.len(), 3);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn write_packed_blobs<'a>(
+        &mut self,
+        blobs: impl IntoIterator<Item = Option<&'a [u8]>>,
+    ) -> Result<()> {
+        let mut writer = self.take_writer()?;
+        let mut descriptors = Vec::new();
+        let mut next_offset = self.offset;
+        for blob in blobs {
+            let Some(blob) = blob else {
+                descriptors.push(BlobDescriptor::Null);
+                continue;
+            };
+            let (descriptor, following_offset) =
+                packed_descriptor(self.blob_id, next_offset, blob.len() as u64)?;
+            if !blob.is_empty() {
+                writer.write_all(blob).await?;
+            }
+            descriptors.push(descriptor);
+            next_offset = following_offset;
+        }
+        self.writer = Some(writer);
+        self.offset = next_offset;
+        self.values.extend(descriptors);
+        Ok(())
+    }
+
     pub(crate) async fn write_blob_bytes(&mut self, bytes: &[u8]) -> Result<BlobDescriptor> {
         let size = bytes.len() as u64;
         let offset = self.offset;
-        self.writer.write_all(bytes).await?;
+        let mut writer = self.take_writer()?;
+        writer.write_all(bytes).await?;
+        self.writer = Some(writer);
         self.record_written_blob(offset, size)
     }
 
@@ -796,28 +862,32 @@ impl PackedBlobWriter {
     ) -> Result<BlobDescriptor> {
         let size = range.len() as u64;
         let offset = self.offset;
-        self.writer.copy_range_from_reader(reader, range).await?;
+        let mut writer = self.take_writer()?;
+        writer.copy_range_from_reader(reader, range).await?;
+        self.writer = Some(writer);
         self.record_written_blob(offset, size)
     }
 
     fn record_written_blob(&mut self, offset: u64, size: u64) -> Result<BlobDescriptor> {
-        self.offset = self.offset.checked_add(size).ok_or_else(|| {
-            Error::invalid_input(format!(
-                "Packed blob writer offset overflowed: offset={offset}, size={size}"
-            ))
-        })?;
-        let value = BlobDescriptor::Packed {
-            blob_id: self.blob_id,
-            offset,
-            size,
-        };
+        let (value, next_offset) = packed_descriptor(self.blob_id, offset, size)?;
+        self.offset = next_offset;
         self.values.push(value.clone());
         Ok(value)
     }
 
+    fn take_writer(&mut self) -> Result<Box<dyn Writer>> {
+        self.writer.take().ok_or_else(|| {
+            Error::io(format!(
+                "Packed blob writer for '{}' has no active upload",
+                self.path
+            ))
+        })
+    }
+
     /// Finish the packed sidecar and return descriptors in write order.
     pub async fn finish(mut self) -> Result<Vec<BlobDescriptor>> {
-        Writer::shutdown(self.writer.as_mut()).await?;
+        let mut writer = self.take_writer()?;
+        Writer::shutdown(writer.as_mut()).await?;
         let object_size = self.object_store.size(&self.path).await?;
         validate_range(0, self.offset, object_size, "Packed blob")?;
         Ok(self.values)
@@ -1010,13 +1080,105 @@ impl BlobArrayBuilder {
 
 #[cfg(test)]
 mod tests {
+    use std::future::Future;
+    use std::io;
     use std::num::NonZeroUsize;
+    use std::pin::Pin;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::task::{Context, Poll};
 
     use super::*;
     use arrow_array::cast::AsArray;
     use arrow_array::{Array, StringArray};
     use arrow_schema::Schema as ArrowSchema;
+    use async_trait::async_trait;
+    use futures::task::noop_waker;
     use lance_core::utils::tempfile::TempDir;
+    use lance_io::object_writer::WriteResult;
+    use tokio::io::AsyncWrite;
+
+    #[derive(Clone, Copy)]
+    enum WriteTerminal {
+        Error,
+        Pending,
+    }
+
+    struct PartialWriter {
+        bytes_before_terminal: usize,
+        terminal: WriteTerminal,
+        bytes_written: Arc<AtomicUsize>,
+        dropped: Arc<AtomicBool>,
+    }
+
+    impl AsyncWrite for PartialWriter {
+        fn poll_write(
+            mut self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            bytes: &[u8],
+        ) -> Poll<io::Result<usize>> {
+            if self.bytes_before_terminal > 0 {
+                let written = self.bytes_before_terminal.min(bytes.len());
+                self.bytes_before_terminal -= written;
+                self.bytes_written.fetch_add(written, Ordering::SeqCst);
+                return Poll::Ready(Ok(written));
+            }
+            match self.terminal {
+                WriteTerminal::Error => {
+                    Poll::Ready(Err(io::Error::other("injected write failure")))
+                }
+                WriteTerminal::Pending => Poll::Pending,
+            }
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[async_trait]
+    impl Writer for PartialWriter {
+        async fn tell(&mut self) -> Result<usize> {
+            Ok(self.bytes_written.load(Ordering::SeqCst))
+        }
+
+        async fn shutdown(&mut self) -> Result<WriteResult> {
+            Ok(WriteResult::default())
+        }
+    }
+
+    impl Drop for PartialWriter {
+        fn drop(&mut self) {
+            self.dropped.store(true, Ordering::SeqCst);
+        }
+    }
+
+    fn partial_writer(
+        terminal: WriteTerminal,
+    ) -> (Box<dyn Writer>, Arc<AtomicUsize>, Arc<AtomicBool>) {
+        let bytes_written = Arc::new(AtomicUsize::new(0));
+        let dropped = Arc::new(AtomicBool::new(false));
+        (
+            Box::new(PartialWriter {
+                bytes_before_terminal: 2,
+                terminal,
+                bytes_written: bytes_written.clone(),
+                dropped: dropped.clone(),
+            }),
+            bytes_written,
+            dropped,
+        )
+    }
+
+    fn cancel_pending<F: Future>(future: F) {
+        let mut future = Box::pin(future);
+        let waker = noop_waker();
+        let mut context = Context::from_waker(&waker);
+        assert!(future.as_mut().poll(&mut context).is_pending());
+    }
 
     #[test]
     fn test_field_metadata() {
@@ -1265,4 +1427,106 @@ mod tests {
         let column = builder.finish().unwrap();
         assert_eq!(column.array().len(), 3);
     }
+
+    #[tokio::test]
+    async fn test_packed_blob_writer_bulk_bytes() {
+        let temp_dir = TempDir::default();
+        let data_dir = Path::from_absolute_path(temp_dir.std_path().join("data")).unwrap();
+        let data_file_path = data_dir.join("data-file.lance");
+        let mut writer = PackedBlobWriter::try_new(ObjectStore::local(), data_file_path, 7)
+            .await
+            .unwrap();
+
+        writer
+            .write_packed_blobs([
+                Some(b"a".as_slice()),
+                Some(b"".as_slice()),
+                None,
+                Some(b"bc".as_slice()),
+            ])
+            .await
+            .unwrap();
+
+        assert_eq!(
+            writer.finish().await.unwrap(),
+            vec![
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 0,
+                    size: 1,
+                },
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 1,
+                    size: 0,
+                },
+                BlobDescriptor::Null,
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 1,
+                    size: 2,
+                },
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_packed_blob_writer_bulk_drops_after_partial_write_error() {
+        let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Error);
+        let previous_descriptor = BlobDescriptor::Packed {
+            blob_id: 7,
+            offset: 0,
+            size: 3,
+        };
+        let mut writer = PackedBlobWriter {
+            object_store: ObjectStore::local(),
+            path: Path::from("packed.blob"),
+            blob_id: 7,
+            writer: Some(partial_writer),
+            offset: 3,
+            values: vec![previous_descriptor.clone()],
+        };
+
+        let error = writer
+            .write_packed_blobs([Some(b"abcdef".as_slice())])
+            .await
+            .unwrap_err();
+
+        assert!(matches!(error, Error::IO { .. }));
+        assert!(error.to_string().contains("injected write failure"));
+        assert_eq!(bytes_written.load(Ordering::SeqCst), 2);
+        assert!(dropped.load(Ordering::SeqCst));
+        assert!(writer.writer.is_none());
+        assert_eq!(writer.offset, 3);
+        assert_eq!(writer.values, vec![previous_descriptor]);
+        let retry_error = writer.write_blob(b"retry").await.unwrap_err();
+        assert!(matches!(retry_error, Error::IO { .. }));
+        assert!(retry_error.to_string().contains("no active upload"));
+    }
+
+    #[test]
+    fn test_packed_blob_writer_bulk_drops_if_cancelled() {
+        let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Pending);
+        let previous_descriptor = BlobDescriptor::Packed {
+            blob_id: 7,
+            offset: 0,
+            size: 3,
+        };
+        let mut writer = PackedBlobWriter {
+            object_store: ObjectStore::local(),
+            path: Path::from("packed.blob"),
+            blob_id: 7,
+            writer: Some(partial_writer),
+            offset: 3,
+            values: vec![previous_descriptor.clone()],
+        };
+
+        cancel_pending(writer.write_packed_blobs([Some(b"abcdef".as_slice())]));
+
+        assert_eq!(bytes_written.load(Ordering::SeqCst), 2);
+        assert!(dropped.load(Ordering::SeqCst));
+        assert!(writer.writer.is_none());
+        assert_eq!(writer.offset, 3);
+        assert_eq!(writer.values, vec![previous_descriptor]);
+    }
 }
diff --git a/rust/lance/src/datafusion/dataframe.rs b/rust/lance/src/datafusion/dataframe.rs
index 00db9920bf9..3009a24a696 100644
--- a/rust/lance/src/datafusion/dataframe.rs
+++ b/rust/lance/src/datafusion/dataframe.rs
@@ -1,10 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{
-    any::Any,
-    sync::{Arc, Mutex},
-};
+use std::sync::{Arc, Mutex};
 
 use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
@@ -42,6 +39,7 @@ pub struct LanceTableProvider {
     row_id_idx: Option<usize>,
     row_addr_idx: Option<usize>,
     ordered: bool,
+    blob_handling: Option<lance_core::datatypes::BlobHandling>,
 }
 
 impl LanceTableProvider {
@@ -72,9 +70,19 @@ impl LanceTableProvider {
             row_id_idx,
             row_addr_idx,
             ordered,
+            blob_handling: None,
         }
     }
 
+    /// Overrides how blob columns are read during [`TableProvider::scan`].
+    ///
+    /// When unset, the underlying dataset scan uses its default
+    /// [`BlobHandling`](lance_core::datatypes::BlobHandling) policy.
+    pub fn with_blob_handling(mut self, handling: lance_core::datatypes::BlobHandling) -> Self {
+        self.blob_handling = Some(handling);
+        self
+    }
+
     pub fn dataset(&self) -> Arc<Dataset> {
         self.dataset.clone()
     }
@@ -82,10 +90,6 @@ impl LanceTableProvider {
 
 #[async_trait]
 impl TableProvider for LanceTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.full_schema.clone()
     }
@@ -102,6 +106,10 @@ impl TableProvider for LanceTableProvider {
         limit: Option<usize>,
     ) -> datafusion::common::Result<Arc<dyn ExecutionPlan>> {
         let mut scan = self.dataset.scan();
+        if let Some(handling) = self.blob_handling.clone() {
+            scan.blob_handling(handling);
+        }
+
         match projection {
             Some(projection) if projection.is_empty() => {
                 scan.empty_project()?;
@@ -166,13 +174,7 @@ pub trait SessionContextExt {
         with_row_id: bool,
         with_row_addr: bool,
     ) -> datafusion::common::Result<DataFrame>;
-    /// Creates a DataFrame for reading a Lance dataset without ordering
-    fn read_lance_unordered(
-        &self,
-        dataset: Arc<Dataset>,
-        with_row_id: bool,
-        with_row_addr: bool,
-    ) -> datafusion::common::Result<DataFrame>;
+
     /// Creates a DataFrame for reading a stream of data
     ///
     /// This dataframe may only be queried once, future queries will fail
@@ -232,20 +234,6 @@ impl SessionContextExt for SessionContext {
         )))
     }
 
-    fn read_lance_unordered(
-        &self,
-        dataset: Arc<Dataset>,
-        with_row_id: bool,
-        with_row_addr: bool,
-    ) -> datafusion::common::Result<DataFrame> {
-        self.read_table(Arc::new(LanceTableProvider::new_with_ordering(
-            dataset,
-            with_row_id,
-            with_row_addr,
-            false,
-        )))
-    }
-
     fn read_one_shot(
         &self,
         data: SendableRecordBatchStream,
diff --git a/rust/lance/src/datafusion/logical_plan.rs b/rust/lance/src/datafusion/logical_plan.rs
index a9fe0ed7750..039aa75864f 100644
--- a/rust/lance/src/datafusion/logical_plan.rs
+++ b/rust/lance/src/datafusion/logical_plan.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{any::Any, borrow::Cow, sync::Arc};
+use std::{borrow::Cow, sync::Arc};
 
 use arrow_schema::Schema as ArrowSchema;
 use async_trait::async_trait;
@@ -19,10 +19,6 @@ use crate::Dataset;
 
 #[async_trait]
 impl TableProvider for Dataset {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> Arc<ArrowSchema> {
         Arc::new(self.schema().into())
     }
@@ -167,17 +163,11 @@ mod tests {
 
         // DataFusion will create a cooperative execution plan, so we need to get its inner plan
         let physical_plan = physical_plan
-            .as_any()
             .downcast_ref::<CooperativeExec>()
             .unwrap()
             .children()[0];
 
-        assert!(
-            physical_plan
-                .as_any()
-                .downcast_ref::<LanceScanExec>()
-                .is_some()
-        );
+        assert!(physical_plan.downcast_ref::<LanceScanExec>().is_some());
 
         let expected_fields = schema
             .fields()
diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs
index 470c6873dc7..9f90a4ba1d5 100644
--- a/rust/lance/src/dataset.rs
+++ b/rust/lance/src/dataset.rs
@@ -37,6 +37,7 @@ use lance_io::object_store::{
     WrappingObjectStore,
 };
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+use lance_io::traits::{WriteExt, Writer};
 use lance_io::utils::{
     CachedFileSize, read_last_block, read_message, read_metadata_offset, read_struct,
 };
@@ -60,7 +61,7 @@ use roaring::RoaringBitmap;
 use rowids::get_row_id_index;
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::Debug;
 use std::num::NonZero;
 use std::ops::Range;
@@ -80,9 +81,10 @@ pub mod index;
 pub mod mem_wal;
 mod metadata;
 pub mod optimize;
+pub(crate) mod overlay;
 pub mod progress;
 pub mod refs;
-pub(crate) mod rowids;
+pub mod rowids;
 pub mod scanner;
 mod schema_evolution;
 pub mod sql;
@@ -117,7 +119,10 @@ use crate::io::commit::{
 use crate::session::Session;
 use crate::utils::temporal::{SystemTime, timestamp_to_nanos, utc_now};
 use crate::{Error, Result};
-pub use blob::{BlobFile, ReadBlob, ReadBlobsBuilder, ReadBlobsStream};
+pub use blob::{
+    BlobFile, BlobRangeRequest, BlobReadRange, ReadBlob, ReadBlobRange, ReadBlobRangesBuilder,
+    ReadBlobRangesStream, ReadBlobsBuilder, ReadBlobsStream,
+};
 use hash_joiner::HashJoiner;
 pub use lance_core::ROW_ID;
 use lance_core::box_error;
@@ -229,6 +234,23 @@ impl From<&Manifest> for Version {
     }
 }
 
+/// The transaction that produced a version of the dataset, along with the
+/// version's commit timestamp.
+///
+/// Returned by [`Dataset::read_version_transaction`], which reads this
+/// information directly from storage without checking out the version.
+#[derive(Debug, Clone)]
+pub struct VersionTransaction {
+    /// Version number.
+    pub version: u64,
+
+    /// Timestamp the version was committed, in UTC.
+    pub timestamp: DateTime<Utc>,
+
+    /// The transaction that produced this version, if one was recorded.
+    pub transaction: Option<Transaction>,
+}
+
 /// Customize read behavior of a dataset.
 #[derive(Clone, Debug)]
 pub struct ReadParams {
@@ -599,7 +621,7 @@ impl Dataset {
             return Ok(self.clone());
         }
 
-        let manifest = Self::load_manifest(
+        let manifest = Self::get_manifest(
             self.object_store.as_ref(),
             &manifest_location,
             &new_location.uri,
@@ -625,7 +647,7 @@ impl Dataset {
             self.object_store.clone(),
             new_location.path,
             new_location.uri,
-            Arc::new(manifest),
+            manifest,
             manifest_location,
             self.session.clone(),
             self.commit_handler.clone(),
@@ -723,6 +745,7 @@ impl Dataset {
             let ds_index_cache = session.index_cache.for_dataset(uri);
             let metadata_key = crate::session::index_caches::IndexMetadataKey {
                 version: manifest_location.version,
+                store_identity: &object_store.store_prefix,
             };
             ds_index_cache
                 .insert_with_key(&metadata_key, Arc::new(indices))
@@ -765,6 +788,11 @@ impl Dataset {
         uri: &str,
         session: &Session,
     ) -> Result<Arc<Manifest>> {
+        if manifest_location.size.is_none() {
+            return Ok(Arc::new(
+                Self::load_manifest(object_store, manifest_location, uri, session).await?,
+            ));
+        }
         let metadata_cache = session.metadata_cache.for_dataset(uri);
         let manifest_key = ManifestKey {
             version: manifest_location.version,
@@ -1181,43 +1209,146 @@ impl Dataset {
             return Ok(Some((*transaction).clone()));
         }
 
+        let transaction = self
+            .read_transaction_from_storage(&self.manifest, &self.manifest_location)
+            .await?;
+
+        if let Some(tx) = transaction.as_ref() {
+            self.metadata_cache
+                .insert_with_key(&transaction_key, Arc::new(tx.clone()))
+                .await;
+        }
+        Ok(transaction)
+    }
+
+    /// Read the transaction recorded by `manifest` directly from storage,
+    /// without consulting or populating any session cache.
+    async fn read_transaction_from_storage(
+        &self,
+        manifest: &Manifest,
+        manifest_location: &ManifestLocation,
+    ) -> Result<Option<Transaction>> {
         // Prefer inline transaction from manifest when available
-        let transaction = if let Some(pos) = self.manifest.transaction_section {
-            let reader = if let Some(size) = self.manifest_location.size {
-                self.object_store
-                    .open_with_size(&self.manifest_location.path, size as usize)
-                    .await?
-            } else {
-                self.object_store.open(&self.manifest_location.path).await?
+        if let Some(pos) = manifest.transaction_section {
+            let reader = match manifest_location.size {
+                Some(size) => {
+                    self.object_store
+                        .open_with_size(&manifest_location.path, size as usize)
+                        .await?
+                }
+                None => self.object_store.open(&manifest_location.path).await?,
             };
 
-            let tx: pb::Transaction = read_message(reader.as_ref(), pos).await?;
-            Transaction::try_from(tx).map(Some)?
-        } else if let Some(path) = &self.manifest.transaction_file {
+            // A concurrent overwrite can leave the listed size too small; retry
+            // once with the true size.
+            let tx: pb::Transaction = match read_message(reader.as_ref(), pos).await {
+                Err(e)
+                    if manifest_location.size.is_some()
+                        && e.to_string().contains("file size is too small") =>
+                {
+                    let reader = self.object_store.open(&manifest_location.path).await?;
+                    read_message(reader.as_ref(), pos).await?
+                }
+                other => other?,
+            };
+            Transaction::try_from(tx).map(Some)
+        } else if let Some(path) = &manifest.transaction_file {
             // Fallback: read external transaction file if present
             let path = self.transactions_dir().join(path.as_str());
             let data = self.object_store.inner.get(&path).await?.bytes().await?;
             let transaction = lance_table::format::pb::Transaction::decode(data)?;
-            Transaction::try_from(transaction).map(Some)?
+            Transaction::try_from(transaction).map(Some)
         } else {
-            None
-        };
+            Ok(None)
+        }
+    }
 
-        if let Some(tx) = transaction.as_ref() {
-            self.metadata_cache
-                .insert_with_key(&transaction_key, Arc::new(tx.clone()))
-                .await;
+    /// Read the transaction (if any) and commit timestamp of a version of the
+    /// dataset. `version` is a version number on this dataset's current branch.
+    ///
+    /// Reads the version's manifest transiently: no historical `Dataset` is
+    /// constructed, no `IndexSection` is decoded, and no session cache is read
+    /// or written, so scanning many historical versions does not fill the
+    /// shared caches.
+    ///
+    /// Returns an error if the version does not exist (for example, if it has
+    /// been cleaned up).
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use lance::{Dataset, Result};
+    /// # async fn example(dataset: &Dataset) -> Result<()> {
+    /// let record = dataset.read_version_transaction(5).await?;
+    /// let committed_at = record.timestamp;
+    /// let operation = record.transaction.as_ref().map(|t| t.operation.name());
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn read_version_transaction(&self, version: u64) -> Result<VersionTransaction> {
+        // Resolve against this dataset's current branch.
+        let manifest_location = self
+            .commit_handler
+            .resolve_version_location(&self.base, version, &self.object_store.inner)
+            .await?;
+
+        // Keep the DatasetNotFound variant callers expect for a missing version.
+        let manifest = read_manifest(
+            &self.object_store,
+            &manifest_location.path,
+            manifest_location.size,
+        )
+        .await
+        .map_err(|e| match &e {
+            Error::NotFound { uri, .. } => Error::dataset_not_found(uri.clone(), box_error(e)),
+            _ => e,
+        })?;
+
+        // The resolved manifest must belong to this dataset's branch. A
+        // mismatch means the commit handler resolved against a different chain
+        // (for example an external manifest store that ignores
+        // branch-qualified paths); error loudly rather than hand back another
+        // branch's transaction.
+        if manifest.branch != self.manifest.branch {
+            return Err(Error::internal(format!(
+                "reading version {} on branch '{}' resolved a manifest belonging to branch '{}'",
+                version,
+                refs::normalize_branch(self.manifest.branch.as_deref()),
+                refs::normalize_branch(manifest.branch.as_deref()),
+            )));
         }
-        Ok(transaction)
+
+        let transaction = self
+            .read_transaction_from_storage(&manifest, &manifest_location)
+            .await?;
+
+        Ok(VersionTransaction {
+            version: manifest.version,
+            timestamp: manifest.timestamp(),
+            transaction,
+        })
     }
 
     /// Read the transaction file for this version of the dataset.
     ///
     /// If there was no transaction file written for this version of the dataset
     /// then this will return None.
+    ///
+    /// Does not populate the session caches; see
+    /// [`Self::read_version_transaction`].
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use lance::{Dataset, Result};
+    /// # async fn example(dataset: &Dataset) -> Result<()> {
+    /// let transaction = dataset.read_transaction_by_version(5).await?;
+    /// let operation = transaction.as_ref().map(|t| t.operation.name());
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn read_transaction_by_version(&self, version: u64) -> Result<Option<Transaction>> {
-        let dataset_version = self.checkout_version(version).await?;
-        dataset_version.read_transaction().await
+        Ok(self.read_version_transaction(version).await?.transaction)
     }
 
     /// List transactions for the dataset, up to a maximum number.
@@ -1604,11 +1735,30 @@ impl Dataset {
     }
 
     /// Take [BlobFile] by row IDs.
+    ///
+    /// The returned vector has one element per row ID. Null blob values are
+    /// represented as `None`; valid empty blobs return a `BlobFile` with size
+    /// zero.
+    ///
+    /// ```
+    /// # use std::sync::Arc;
+    /// # use lance::dataset::Dataset;
+    /// # use lance::Result;
+    /// # async fn example(dataset: Arc<Dataset>) -> Result<()> {
+    /// let blobs = dataset.take_blobs(&[42], "images").await?;
+    /// match &blobs[0] {
+    ///     None => { /* The selected blob is null. */ }
+    ///     Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ }
+    ///     Some(blob) => { let _size = blob.size(); }
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn take_blobs(
         self: &Arc<Self>,
         row_ids: &[u64],
         column: impl AsRef<str>,
-    ) -> Result<Vec<BlobFile>> {
+    ) -> Result<Vec<Option<BlobFile>>> {
         blob::take_blobs(self, row_ids, column.as_ref()).await
     }
 
@@ -1618,21 +1768,57 @@ impl Dataset {
     /// Use this method when you already have row addresses, for example from
     /// a scan with `with_row_address()`. For row IDs (stable identifiers), use
     /// [`Self::take_blobs`]. For row indices (offsets), use
-    /// [`Self::take_blobs_by_indices`].
+    /// [`Self::take_blobs_by_indices`]. The result has the same null and empty
+    /// blob representation as [`Self::take_blobs`].
+    ///
+    /// ```
+    /// # use std::sync::Arc;
+    /// # use lance::dataset::Dataset;
+    /// # use lance::Result;
+    /// # async fn example(dataset: Arc<Dataset>, row_address: u64) -> Result<()> {
+    /// let blobs = dataset
+    ///     .take_blobs_by_addresses(&[row_address], "images")
+    ///     .await?;
+    /// match &blobs[0] {
+    ///     None => { /* The selected blob is null. */ }
+    ///     Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ }
+    ///     Some(blob) => { let _size = blob.size(); }
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn take_blobs_by_addresses(
         self: &Arc<Self>,
         row_addrs: &[u64],
         column: impl AsRef<str>,
-    ) -> Result<Vec<BlobFile>> {
+    ) -> Result<Vec<Option<BlobFile>>> {
         blob::take_blobs_by_addresses(self, row_addrs, column.as_ref()).await
     }
 
     /// Take [BlobFile] by row indices (offsets in the dataset).
+    ///
+    /// The result has the same null and empty blob representation as
+    /// [`Self::take_blobs`].
+    ///
+    /// ```
+    /// # use std::sync::Arc;
+    /// # use lance::dataset::Dataset;
+    /// # use lance::Result;
+    /// # async fn example(dataset: Arc<Dataset>) -> Result<()> {
+    /// let blobs = dataset.take_blobs_by_indices(&[0], "images").await?;
+    /// match &blobs[0] {
+    ///     None => { /* The selected blob is null. */ }
+    ///     Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ }
+    ///     Some(blob) => { let _size = blob.size(); }
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn take_blobs_by_indices(
         self: &Arc<Self>,
         row_indices: &[u64],
         column: impl AsRef<str>,
-    ) -> Result<Vec<BlobFile>> {
+    ) -> Result<Vec<Option<BlobFile>>> {
         let fragments = self.get_fragments();
         let row_addrs = row_offsets_to_row_addresses(&fragments, row_indices).await?;
         blob::take_blobs_by_addresses(self, &row_addrs, column.as_ref()).await
@@ -1643,7 +1829,9 @@ impl Dataset {
     /// This API complements [`Self::take_blobs`]. `take_blobs` returns
     /// [`BlobFile`] handles for caller-driven random access, while
     /// `read_blobs` builds a streaming read plan for sequential or batched blob
-    /// retrieval.
+    /// retrieval. Every selected row produces one result: null blob values have
+    /// `ReadBlob::data` set to `None`, while valid empty blobs contain an empty
+    /// buffer.
     ///
     /// ```rust
     /// # use std::sync::Arc;
@@ -1670,6 +1858,38 @@ impl Dataset {
         ))
     }
 
+    /// Create a planned reader for row-specific blob-local byte ranges.
+    ///
+    /// Each [`BlobRangeRequest`] contains both its row selector and byte range,
+    /// so requests can be repeated or reordered without coordinating parallel
+    /// selector and range lists. Every request produces one result. A null blob
+    /// has `ReadBlobRange::data` set to `None`; an empty range on a non-null blob
+    /// contains an empty buffer.
+    ///
+    /// ```rust
+    /// # use std::sync::Arc;
+    /// # use lance::dataset::{BlobRangeRequest, Dataset};
+    /// # use lance::Result;
+    /// # async fn example(dataset: Arc<Dataset>) -> Result<()> {
+    /// let ranges = dataset
+    ///     .read_blob_ranges("images")?
+    ///     .with_row_indices([
+    ///         BlobRangeRequest::new(7, 0, 1024),
+    ///         BlobRangeRequest::new(7, 4096, 1024),
+    ///     ])
+    ///     .execute()
+    ///     .await?;
+    /// # let _ = ranges;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn read_blob_ranges(
+        self: &Arc<Self>,
+        column: impl AsRef<str>,
+    ) -> Result<ReadBlobRangesBuilder> {
+        Ok(ReadBlobRangesBuilder::new(self.read_blobs(column)?))
+    }
+
     /// Get a stream of batches based on iterator of ranges of row numbers.
     ///
     /// This is an experimental API. It may change at any time.
@@ -1711,26 +1931,7 @@ impl Dataset {
                     ));
                 }
 
-                let selected_fragment_ids = fragment_ids.iter().copied().collect::<BTreeSet<_>>();
-                let selected_fragments = self
-                    .get_fragments()
-                    .into_iter()
-                    .filter(|fragment| selected_fragment_ids.contains(&(fragment.id() as u32)))
-                    .collect::<Vec<_>>();
-
-                if selected_fragments.len() != selected_fragment_ids.len() {
-                    let present_fragment_ids = selected_fragments
-                        .iter()
-                        .map(|fragment| fragment.id() as u32)
-                        .collect::<HashSet<_>>();
-                    let missing_fragment_ids = selected_fragment_ids
-                        .into_iter()
-                        .filter(|fragment_id| !present_fragment_ids.contains(fragment_id))
-                        .collect::<Vec<_>>();
-                    return Err(Error::invalid_input(format!(
-                        "Dataset::sample received fragment ids that are not part of the current dataset version: {missing_fragment_ids:?}",
-                    )));
-                }
+                let selected_fragments = self.get_fragments_from_ids(fragment_ids)?;
 
                 let num_rows = stream::iter(selected_fragments.iter().cloned())
                     .map(|fragment| async move { fragment.count_rows(None).await })
@@ -2304,6 +2505,13 @@ impl Dataset {
         }
     }
 
+    /// The `ObjectStoreParams` this dataset was opened with, or `None` when
+    /// opened without explicit params. Lets a caller re-open a derived path
+    /// (e.g. a MemWAL SSTable) with the same store this dataset used.
+    pub fn store_params(&self) -> Option<&ObjectStoreParams> {
+        self.store_params.as_deref()
+    }
+
     pub(crate) async fn object_store_for_data_file(
         &self,
         data_file: &DataFile,
@@ -2540,41 +2748,121 @@ impl Dataset {
         &self.manifest.fragments
     }
 
-    // Gets a filtered list of fragments from ids in O(N) time instead of using
-    // `get_fragment` which would require O(N^2) time.
-    pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec<Option<FileFragment>> {
-        let mut fragments = Vec::with_capacity(ordered_ids.len());
-        let mut id_iter = ordered_ids.iter();
-        let mut id = id_iter.next();
-        // This field is just used to assert the ids are in order
-        let mut last_id: i64 = -1;
-        for frag in self.manifest.fragments.iter() {
-            let mut the_id = if let Some(id) = id { *id } else { break };
-            // Assert the given ids are, in fact, in order
-            assert!(the_id as i64 > last_id);
-            // For any IDs we've passed we can assume that no fragment exists any longer
-            // with that ID.
-            while the_id < frag.id as u32 {
-                fragments.push(None);
-                last_id = the_id as i64;
-                id = id_iter.next();
-                the_id = if let Some(id) = id { *id } else { break };
-            }
+    pub(crate) fn normalize_fragment_ids(fragment_ids: &[u32]) -> Vec<u32> {
+        let mut ids = fragment_ids.to_vec();
+        ids.sort_unstable();
+        ids.dedup();
+        ids
+    }
 
-            if the_id == frag.id as u32 {
-                fragments.push(Some(FileFragment::new(
-                    Arc::new(self.clone()),
-                    frag.clone(),
-                )));
-                last_id = the_id as i64;
-                id = id_iter.next();
-            }
+    pub(crate) fn get_fragments_from_ids(&self, fragment_ids: &[u32]) -> Result<Vec<FileFragment>> {
+        let ordered_ids = Self::normalize_fragment_ids(fragment_ids);
+        let fragments = self.get_frags_from_ordered_ids(&ordered_ids);
+        if let Some(missing_id) = fragments
+            .iter()
+            .zip(ordered_ids.iter())
+            .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id))
+        {
+            return Err(Error::invalid_input(format!(
+                "Unknown fragment id {missing_id} in fragment filter; not part of the current dataset version"
+            )));
         }
-        fragments
+
+        Ok(fragments.into_iter().flatten().collect())
+    }
+
+    pub(crate) fn get_existing_fragments_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Vec<FileFragment> {
+        let ordered_ids = Self::normalize_fragment_ids(fragment_ids);
+        self.get_frags_from_ordered_ids(&ordered_ids)
+            .into_iter()
+            .flatten()
+            .collect()
+    }
+
+    pub(crate) fn get_fragment_metadata_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Result<Vec<Fragment>> {
+        Ok(self
+            .get_fragments_from_ids(fragment_ids)?
+            .into_iter()
+            .map(|fragment| fragment.metadata().clone())
+            .collect())
+    }
+
+    pub(crate) fn get_existing_fragment_metadata_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Vec<Fragment> {
+        self.get_existing_fragments_from_ids(fragment_ids)
+            .into_iter()
+            .map(|fragment| fragment.metadata().clone())
+            .collect()
+    }
+
+    pub(crate) async fn count_rows_in_fragments(&self, fragment_ids: &[u32]) -> Result<usize> {
+        let fragments = self.get_fragments_from_ids(fragment_ids)?;
+        self.count_rows_in_resolved_fragments(fragments).await
+    }
+
+    pub(crate) async fn count_rows_in_existing_fragments(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Result<usize> {
+        let fragments = self.get_existing_fragments_from_ids(fragment_ids);
+        self.count_rows_in_resolved_fragments(fragments).await
+    }
+
+    async fn count_rows_in_resolved_fragments(
+        &self,
+        fragments: Vec<FileFragment>,
+    ) -> Result<usize> {
+        let counts = stream::iter(fragments)
+            .map(|fragment| async move { fragment.count_rows(None).await })
+            .buffer_unordered(16)
+            .try_collect::<Vec<_>>()
+            .await?;
+        Ok(counts.iter().sum())
+    }
+
+    /// Resolves fragments for the given ids without scanning the manifest.
+    ///
+    /// The ids do not need to be sorted or deduplicated. Each id is resolved
+    /// independently via the fragment bitmap.
+    pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec<Option<FileFragment>> {
+        let dataset = Arc::new(self.clone());
+        ordered_ids
+            .iter()
+            .map(|id| {
+                if !self.fragment_bitmap.contains(*id) {
+                    return None;
+                }
+                let fragment_index = self.fragment_bitmap.rank(*id) as usize - 1;
+                let fragment = self.manifest.fragments.get(fragment_index)?;
+                debug_assert_eq!(
+                    fragment.id, *id as u64,
+                    "fragment_bitmap rank({id}) resolved to fragment {}, but fragment_bitmap and manifest.fragments are expected to stay in sync",
+                    fragment.id
+                );
+                Some(FileFragment::new(dataset.clone(), fragment.clone()))
+            })
+            .collect()
     }
 
     // This method filters deleted items from `addr_or_ids` using `addrs` as a reference
     async fn filter_addr_or_ids(&self, addr_or_ids: &[u64], addrs: &[u64]) -> Result<Vec<u64>> {
+        // The final zip pairs these positionally; misalignment must fail
+        // loud rather than truncate.
+        if addr_or_ids.len() != addrs.len() {
+            return Err(Error::internal(format!(
+                "filter_addr_or_ids: addr_or_ids has {} entries but addrs has {}",
+                addr_or_ids.len(),
+                addrs.len()
+            )));
+        }
         if addrs.is_empty() {
             return Ok(Vec::new());
         }
@@ -2686,17 +2974,24 @@ impl Dataset {
     }
 
     pub(crate) async fn filter_deleted_ids(&self, ids: &[u64]) -> Result<Vec<u64>> {
-        let addresses = if let Some(row_id_index) = get_row_id_index(self).await? {
-            let addresses = ids
-                .iter()
-                .filter_map(|id| row_id_index.get(*id).map(|address| address.into()))
-                .collect::<Vec<_>>();
-            Cow::Owned(addresses)
+        let (ids, addresses) = if let Some(row_id_index) = get_row_id_index(self).await? {
+            // Ids absent from the deletion-aware index are deleted; drop
+            // them from both lists to keep the zip aligned. ids.len() is an
+            // upper bound on the output size, so allocate once up front.
+            let mut live_ids = Vec::with_capacity(ids.len());
+            let mut addresses = Vec::with_capacity(ids.len());
+            for id in ids {
+                if let Some(address) = row_id_index.get(*id) {
+                    live_ids.push(*id);
+                    addresses.push(u64::from(address));
+                }
+            }
+            (Cow::Owned(live_ids), Cow::Owned(addresses))
         } else {
-            Cow::Borrowed(ids)
+            (Cow::Borrowed(ids), Cow::Borrowed(ids))
         };
 
-        self.filter_addr_or_ids(ids, &addresses).await
+        self.filter_addr_or_ids(&ids, &addresses).await
     }
 
     /// Gets the number of files that are so small they don't even have a full
@@ -2774,7 +3069,7 @@ impl Dataset {
                     self.manifest_location.path.clone(),
                     format!(
                         "Duplicate index id {} found in dataset {:?}",
-                        &index.uuid, self.base
+                        index.uuid, self.base
                     ),
                 ));
             }
@@ -2872,14 +3167,24 @@ impl Dataset {
     }
 
     /// Deep clone the target version into a new dataset at target_path.
-    /// This performs a server-side copy of all relevant dataset files (data files,
-    /// deletion files, and any external row-id files) into the target dataset
-    /// without loading data into memory.
+    /// This copies all relevant dataset files (data files, deletion files, and
+    /// index files) into the target dataset without loading data into memory.
+    ///
+    /// The source files are read through this dataset's own object store while the
+    /// copies are written through the target object store built from `store_params`.
+    /// This makes the clone work across accounts/stores (e.g. between two abfss
+    /// accounts): when the source and target stores are the same the copy stays
+    /// server-side, otherwise the data is streamed through this process.
     ///
     /// Parameters:
     /// - `target_path`: the URI string to clone the dataset into.
     /// - `version`: the version cloned from, could be a version number, branch head, or tag.
-    /// - `store_params`: the object store params to use for the new dataset.
+    /// - `store_params`: the object store params for the target dataset (e.g. the
+    ///   credentials of the target account).
+    ///
+    /// Note: external `base_paths` referenced by the source manifest are read through
+    /// this dataset's object store; per-base distinct source credentials are not yet
+    /// supported (see <https://github.com/lance-format/lance/issues/6093>).
     pub async fn deep_clone(
         &mut self,
         target_path: &str,
@@ -2920,7 +3225,12 @@ impl Dataset {
             path
         };
 
-        // TODO: Leverage object store bulk copy for efficient deep_clone
+        // When the source and target live in the same store we can keep the copy
+        // server-side. Otherwise (e.g. cloning across accounts) we stream each file
+        // from the source store to the target store.
+        let same_store = src_ds.object_store.store_prefix == target_store.store_prefix;
+
+        // TODO: Leverage object store bulk copy for efficient same-store deep_clone.
         //
         // All cloud storage providers support batch copy APIs that would provide significant
         // performance improvements. We use single file copy before we have upstream support.
@@ -2930,10 +3240,21 @@ impl Dataset {
         let copy_futures = src_paths
             .iter()
             .map(|(relative_path, base)| {
-                let store = Arc::clone(&target_store);
+                let source_store = Arc::clone(&src_ds.object_store);
+                let target_store = Arc::clone(&target_store);
                 let src_path = build_absolute_path(relative_path, base);
                 let target_path = build_absolute_path(relative_path, &target_base);
-                async move { store.copy(&src_path, &target_path).await.map(|_| ()) }
+                async move {
+                    if same_store {
+                        target_store.copy(&src_path, &target_path).await?;
+                    } else {
+                        let reader = source_store.open(&src_path).await?;
+                        let mut writer = target_store.create(&target_path).await?;
+                        writer.copy_from_reader(reader.as_ref()).await?;
+                        writer.shutdown().await?;
+                    }
+                    Result::Ok(())
+                }
             })
             .collect::<Vec<_>>();
 
@@ -2958,6 +3279,7 @@ impl Dataset {
         let builder = CommitBuilder::new(WriteDestination::Uri(target_path))
             .with_store_params(store_params.clone().unwrap_or_default())
             .with_object_store(target_store.clone())
+            .with_source_store(src_ds.object_store.clone())
             .with_commit_handler(self.commit_handler.clone())
             .with_storage_format(self.manifest.data_storage_format.lance_file_version()?);
         let new_ds = builder.execute(txn).await?;
diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs
index 84c1b8bdcad..747ca7ba2ab 100644
--- a/rust/lance/src/dataset/blob.rs
+++ b/rust/lance/src/dataset/blob.rs
@@ -12,16 +12,19 @@ use std::{
 
 use arrow::array::AsArray;
 use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type};
-use arrow_array::RecordBatch;
-use arrow_array::{Array, ArrayRef};
-use arrow_schema::{DataType as ArrowDataType, Field as ArrowField};
+use arrow_array::{
+    Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, builder::LargeBinaryBuilder,
+};
+use arrow_buffer::{OffsetBuffer, ScalarBuffer};
+use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema};
 use bytes::Bytes;
 use futures::future::BoxFuture;
 use futures::stream::BoxStream;
 use futures::{FutureExt, StreamExt, TryStreamExt, stream};
 use lance_arrow::{
-    BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
-    BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt, r#struct::StructArrayExt,
+    ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+    BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt,
+    list::ListArrayExt, r#struct::StructArrayExt,
 };
 use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
 use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig};
@@ -37,9 +40,9 @@ use crate::blob::{
     is_logical_blob_v2_field, is_prepared_blob_v2_field, validate_prepared_blob_array,
 };
 use arrow_array::StructArray;
-use lance_core::datatypes::{BlobKind, BlobVersion, parse_field_path};
+use lance_core::datatypes::{BlobKind, BlobVersion, Field as LanceField, Schema, parse_field_path};
 use lance_core::utils::blob::blob_path;
-use lance_core::{Error, Result, utils::address::RowAddress};
+use lance_core::{Error, ROW_ADDR, Result, utils::address::RowAddress};
 use lance_io::traits::Reader;
 use lance_io::utils::CachedFileSize;
 
@@ -323,6 +326,9 @@ enum BlobPreprocessFieldKind {
     Struct {
         children: Vec<BlobPreprocessField>,
     },
+    List {
+        child: Box<BlobPreprocessField>,
+    },
     Passthrough,
 }
 
@@ -374,6 +380,17 @@ impl BlobPreprocessField {
             }
         }
 
+        if let ArrowDataType::List(child) | ArrowDataType::LargeList(child) = field.data_type() {
+            let child = Self::new(child.as_ref())?;
+            if child.requires_preprocessing() {
+                return Ok(Self {
+                    kind: BlobPreprocessFieldKind::List {
+                        child: Box::new(child),
+                    },
+                });
+            }
+        }
+
         Ok(Self {
             kind: BlobPreprocessFieldKind::Passthrough,
         })
@@ -659,6 +676,20 @@ impl BlobPreprocessor {
                     self.preprocess_struct_array(array, field.as_ref(), children)
                         .await
                 }
+                BlobPreprocessFieldKind::List { child } => match field.data_type() {
+                    ArrowDataType::List(_) => {
+                        self.preprocess_list_array::<i32>(array, field.as_ref(), child)
+                            .await
+                    }
+                    ArrowDataType::LargeList(_) => {
+                        self.preprocess_list_array::<i64>(array, field.as_ref(), child)
+                            .await
+                    }
+                    _ => Err(Error::internal(format!(
+                        "Blob list preprocessor received non-list field '{}'",
+                        field.name()
+                    ))),
+                },
             }
         }
         .boxed()
@@ -691,10 +722,8 @@ impl BlobPreprocessor {
 
         let mut new_columns = Vec::with_capacity(children.len());
         let mut new_fields = Vec::with_capacity(children.len());
-        for ((child_processor, child_array), child_field) in children
-            .iter()
-            .zip(child_columns.into_iter())
-            .zip(child_fields.iter())
+        for ((child_processor, child_array), child_field) in
+            children.iter().zip(child_columns).zip(child_fields.iter())
         {
             let (new_column, new_field) = self
                 .preprocess_field(child_processor, child_array, child_field)
@@ -716,6 +745,79 @@ impl BlobPreprocessor {
         Ok((Arc::new(struct_array), field))
     }
 
+    async fn preprocess_list_array<O: OffsetSizeTrait>(
+        &mut self,
+        array: ArrayRef,
+        field: &ArrowField,
+        child: &BlobPreprocessField,
+    ) -> Result<(ArrayRef, Arc<ArrowField>)> {
+        let list_arr = array.as_list::<O>();
+        let list_arr = if list_arr.null_count() > 0 {
+            list_arr.filter_garbage_nulls()
+        } else {
+            list_arr.clone()
+        };
+
+        let first_offset = *list_arr
+            .offsets()
+            .first()
+            .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?;
+        let last_offset = *list_arr
+            .offsets()
+            .last()
+            .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?;
+        let values_len = list_arr.values().len();
+        let needs_trim = first_offset != O::zero()
+            || last_offset.to_usize().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "List field '{}' offset does not fit into usize",
+                    field.name()
+                ))
+            })? != values_len;
+
+        let (offsets, values) = if needs_trim {
+            let values = list_arr.trimmed_values();
+            let offsets = list_arr
+                .offsets()
+                .iter()
+                .map(|offset| *offset - first_offset)
+                .collect::<Vec<_>>();
+            (OffsetBuffer::new(ScalarBuffer::from(offsets)), values)
+        } else {
+            (list_arr.offsets().clone(), list_arr.values().clone())
+        };
+
+        let child_field = match field.data_type() {
+            ArrowDataType::List(child_field) | ArrowDataType::LargeList(child_field) => {
+                child_field.clone()
+            }
+            other => {
+                return Err(Error::invalid_input(format!(
+                    "Blob list preprocessor expected list field '{}', got {other}",
+                    field.name()
+                )));
+            }
+        };
+        let (new_values, new_child_field) =
+            self.preprocess_field(child, values, &child_field).await?;
+
+        let list_array = GenericListArray::<O>::try_new(
+            new_child_field,
+            offsets,
+            new_values,
+            list_arr.nulls().cloned(),
+        )?;
+        let field = Arc::new(
+            ArrowField::new(
+                field.name(),
+                list_array.data_type().clone(),
+                field.is_nullable(),
+            )
+            .with_metadata(field.metadata().clone()),
+        );
+        Ok((Arc::new(list_array), field))
+    }
+
     async fn preprocess_blob_array(
         &mut self,
         array: ArrayRef,
@@ -987,7 +1089,9 @@ impl BlobSource {
     /// Drain currently queued requests and submit them as scheduler batches.
     ///
     /// Each loop iteration grabs the queued requests with a short mutex hold and
-    /// immediately releases the lock before any I/O is awaited.
+    /// dispatches them without waiting for earlier batches to finish. Awaiting a
+    /// batch here would hold later, naturally staggered callers behind its I/O.
+    /// [`FileScheduler`] owns the concurrency and backpressure for dispatched I/O.
     async fn drain_pending_reads(self: Arc<Self>, scheduler: FileScheduler) {
         loop {
             let batch = {
@@ -998,7 +1102,10 @@ impl BlobSource {
                 }
                 std::mem::take(&mut pending_reads.requests)
             };
-            fulfill_pending_blob_reads(&scheduler, batch).await;
+            let scheduler = scheduler.clone();
+            tokio::spawn(async move {
+                fulfill_pending_blob_reads(&scheduler, batch).await;
+            });
         }
     }
 }
@@ -1468,13 +1575,80 @@ impl BlobFile {
 pub struct ReadBlob {
     /// Row address of the blob that was read.
     pub row_address: u64,
-    /// Blob payload bytes.
-    pub data: Bytes,
+    /// Blob payload bytes, or `None` when the selected blob value is null.
+    ///
+    /// A valid empty blob is represented as `Some(Bytes::new())`.
+    pub data: Option<Bytes>,
+}
+
+/// A byte range relative to the beginning of one logical blob value.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct BlobReadRange {
+    /// Byte offset from the beginning of the blob value.
+    pub offset: u64,
+    /// Number of bytes to read.
+    pub length: u64,
+}
+
+impl BlobReadRange {
+    /// Create a blob-local byte range from an offset and length.
+    pub const fn new(offset: u64, length: u64) -> Self {
+        Self { offset, length }
+    }
+
+    fn checked_range(self, request_index: usize) -> Result<Range<u64>> {
+        let end = self.offset.checked_add(self.length).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Blob range request {request_index} offset + length overflowed u64: offset={}, length={}",
+                self.offset, self.length
+            ))
+        })?;
+        Ok(self.offset..end)
+    }
+}
+
+/// One row-specific blob range read request.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct BlobRangeRequest {
+    /// Row selector interpreted by the selection method on
+    /// [`ReadBlobRangesBuilder`].
+    pub row: u64,
+    /// Blob-local byte range to read from the selected row.
+    pub range: BlobReadRange,
+}
+
+impl BlobRangeRequest {
+    /// Create a request for `length` bytes at `offset` in the selected row.
+    pub const fn new(row: u64, offset: u64, length: u64) -> Self {
+        Self {
+            row,
+            range: BlobReadRange::new(offset, length),
+        }
+    }
+}
+
+/// Bytes materialized for one request submitted through [`ReadBlobRangesBuilder`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ReadBlobRange {
+    /// Zero-based position of this request in the caller's request list.
+    pub request_index: usize,
+    /// Physical row address of the selected blob value.
+    pub row_address: u64,
+    /// Blob-local range supplied for this request.
+    pub range: BlobReadRange,
+    /// Bytes in `range`, or `None` when the selected blob value is null.
+    ///
+    /// An empty range on a non-null blob is represented as
+    /// `Some(Bytes::new())`.
+    pub data: Option<Bytes>,
 }
 
 /// Stream returned by [`ReadBlobsBuilder::try_into_stream`].
 pub type ReadBlobsStream = BoxStream<'static, Result<ReadBlob>>;
 
+/// Stream returned by [`ReadBlobRangesBuilder::try_into_stream`].
+pub type ReadBlobRangesStream = BoxStream<'static, Result<ReadBlobRange>>;
+
 /// Row selector configured on [`ReadBlobsBuilder`].
 #[derive(Debug, Clone)]
 enum ReadBlobsSelection {
@@ -1559,77 +1733,41 @@ impl ReadBlobsBuilder {
 
     /// Execute the planned blob read and return a stream of blob payloads.
     ///
-    /// The stream yields one [`ReadBlob`] per selected non-null blob row.
+    /// The stream yields one [`ReadBlob`] per selected row. Null blob values
+    /// have `data` set to `None`; valid empty blobs contain an empty buffer.
     pub async fn try_into_stream(self) -> Result<ReadBlobsStream> {
         self.validate()?;
-        let entries = collect_blob_entries_for_selection(
+        let collected = collect_blob_selection_for_selection(
             &self.dataset,
             self.blob_field_id,
             &self.column,
             &self.selection,
         )
         .await?;
-        let expected_selection_indices = entries
-            .iter()
-            .map(|entry| entry.selection_index)
-            .collect::<VecDeque<_>>();
-        let plans = plan_blob_read_plans(entries);
+        let (selection_count, null_blobs) =
+            collect_null_read_blobs(&collected.entries, collected.row_addresses)?;
+        let physical_buffer_size = self.options.io_buffer_size_bytes.unwrap_or_else(|| {
+            SchedulerConfig::max_bandwidth(self.dataset.object_store.as_ref()).io_buffer_size_bytes
+        });
+        let batches = plan_blob_read_batches(collected.entries, None, physical_buffer_size)?;
         let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes));
-        if plans.is_empty() {
-            return Ok(stream::empty().boxed());
-        }
-
-        let plan_stream = stream::iter(plans.into_iter().map(move |plan| {
-            let execution = execution.clone();
-            execute_blob_read_plan(plan, execution)
-        }))
-        .buffer_unordered(self.dataset.object_store.io_parallelism().max(1));
-
-        if !self.options.preserve_order {
-            return Ok(plan_stream
-                .map_ok(|blobs| {
-                    stream::iter(blobs.into_iter().map(|blob| Ok(into_read_blob(blob))))
-                })
-                .try_flatten()
-                .boxed());
-        }
-
-        let mut plan_stream = plan_stream.boxed();
-        let mut expected_selection_indices = expected_selection_indices;
-        let mut ready = BTreeMap::<usize, ReadBlob>::new();
-
-        Ok(stream::poll_fn(move |cx| {
-            loop {
-                let Some(next_selection_index) = expected_selection_indices.front().copied() else {
-                    return Poll::Ready(None);
-                };
-
-                if let Some(blob) = ready.remove(&next_selection_index) {
-                    expected_selection_indices.pop_front();
-                    return Poll::Ready(Some(Ok(blob)));
-                }
-
-                match plan_stream.poll_next_unpin(cx) {
-                    Poll::Ready(Some(Ok(blobs))) => {
-                        for blob in blobs {
-                            ready.insert(blob.selection_index, into_read_blob(blob));
-                        }
-                    }
-                    Poll::Ready(Some(Err(err))) => {
-                        return Poll::Ready(Some(Err(err)));
-                    }
-                    Poll::Ready(None) => {
-                        let err = Error::internal(format!(
-                            "planned blob read stream completed before selection index {} was produced",
-                            next_selection_index
-                        ));
-                        return Poll::Ready(Some(Err(err)));
-                    }
-                    Poll::Pending => return Poll::Pending,
-                }
-            }
+        let non_null_blobs = execute_blob_read_batches_stream(
+            batches,
+            execution,
+            self.dataset.object_store.io_parallelism(),
+            self.options.preserve_order,
+        )
+        .map_ok(|blob| {
+            let selection_index = blob.selection_index;
+            (selection_index, into_read_blob(blob))
         })
-        .boxed())
+        .boxed();
+        Ok(totalize_blob_selection_stream(
+            non_null_blobs,
+            null_blobs,
+            selection_count,
+            self.options.preserve_order,
+        ))
     }
 
     /// Execute the planned blob read and collect the full result in memory.
@@ -1650,6 +1788,150 @@ impl ReadBlobsBuilder {
     }
 }
 
+/// Builder for planned blob-local range reads.
+///
+/// Planning retains `O(request_count)` metadata. [`Self::try_into_stream`]
+/// bounds each scheduler-visible physical batch by the configured I/O buffer
+/// size after accounting for range coalescing, except that one request larger
+/// than the buffer must be materialized by itself. The existing scheduler
+/// separately applies byte backpressure to physical I/O. [`Self::execute`]
+/// additionally retains all returned payload bytes.
+#[derive(Debug, Clone)]
+pub struct ReadBlobRangesBuilder {
+    inner: ReadBlobsBuilder,
+    ranges: Vec<BlobReadRange>,
+}
+
+impl ReadBlobRangesBuilder {
+    pub(crate) fn new(inner: ReadBlobsBuilder) -> Self {
+        Self {
+            inner,
+            ranges: Vec::new(),
+        }
+    }
+
+    /// Read requests whose row values are stable row ids.
+    pub fn with_row_ids(self, requests: impl IntoIterator<Item = BlobRangeRequest>) -> Self {
+        self.with_requests(requests, ReadBlobsSelection::RowIds)
+    }
+
+    /// Read requests whose row values are offsets in dataset order.
+    pub fn with_row_indices(self, requests: impl IntoIterator<Item = BlobRangeRequest>) -> Self {
+        self.with_requests(requests, ReadBlobsSelection::RowIndices)
+    }
+
+    /// Read requests whose row values are physical row addresses.
+    pub fn with_row_addresses(self, requests: impl IntoIterator<Item = BlobRangeRequest>) -> Self {
+        self.with_requests(requests, ReadBlobsSelection::RowAddresses)
+    }
+
+    /// Set the scheduler I/O buffer size used while materializing ranges.
+    pub fn with_io_buffer_size_bytes(mut self, bytes: u64) -> Self {
+        self.inner.options.io_buffer_size_bytes = Some(bytes);
+        self
+    }
+
+    /// Whether results must follow the caller's request order.
+    pub fn preserve_order(mut self, preserve: bool) -> Self {
+        self.inner.options.preserve_order = preserve;
+        self
+    }
+
+    /// Execute the planned range read and return one result per request.
+    ///
+    /// Null blob requests have `data` set to `None`. Empty ranges on non-null
+    /// blobs contain an empty buffer without issuing payload I/O. By default,
+    /// results follow input order; `request_index` identifies the original
+    /// request even when ordering is disabled. Blob-local bounds are not
+    /// evaluated for null values because they have no logical payload length.
+    pub async fn try_into_stream(self) -> Result<ReadBlobRangesStream> {
+        self.validate()?;
+        let collected = collect_blob_selection_for_selection(
+            &self.inner.dataset,
+            self.inner.blob_field_id,
+            &self.inner.column,
+            &self.inner.selection,
+        )
+        .await?;
+        let (selection_count, null_ranges) = collect_null_read_blob_ranges(
+            &collected.entries,
+            collected.row_addresses,
+            &self.ranges,
+        )?;
+        let physical_buffer_size = self.inner.options.io_buffer_size_bytes.unwrap_or_else(|| {
+            SchedulerConfig::max_bandwidth(self.inner.dataset.object_store.as_ref())
+                .io_buffer_size_bytes
+        });
+        let batches =
+            plan_blob_read_batches(collected.entries, Some(&self.ranges), physical_buffer_size)?;
+        let execution = Arc::new(ReadBlobsExecution::new(
+            self.inner.options.io_buffer_size_bytes,
+        ));
+        let non_null_ranges = execute_blob_read_batches_stream(
+            batches,
+            execution,
+            self.inner.dataset.object_store.io_parallelism(),
+            self.inner.options.preserve_order,
+        )
+        .map(|result| {
+            result.and_then(|blob| {
+                let selection_index = blob.selection_index;
+                into_read_blob_range(blob).map(|range| (selection_index, range))
+            })
+        })
+        .boxed();
+        Ok(totalize_blob_selection_stream(
+            non_null_ranges,
+            null_ranges,
+            selection_count,
+            self.inner.options.preserve_order,
+        ))
+    }
+
+    /// Execute the planned range read and collect all returned bytes in memory.
+    pub async fn execute(self) -> Result<Vec<ReadBlobRange>> {
+        self.try_into_stream().await?.try_collect().await
+    }
+
+    fn validate(&self) -> Result<()> {
+        match self.inner.selection {
+            ReadBlobsSelection::None => {
+                return Err(Error::invalid_input(
+                    "ReadBlobRangesBuilder requires requests; call one of with_row_ids, with_row_indices, or with_row_addresses".to_string(),
+                ));
+            }
+            _ if self.inner.options.io_buffer_size_bytes == Some(0) => {
+                return Err(Error::invalid_input(
+                    "ReadBlobRangesBuilder io_buffer_size must be greater than 0".to_string(),
+                ));
+            }
+            _ => {}
+        }
+        for (request_index, range) in self.ranges.iter().enumerate() {
+            range.checked_range(request_index)?;
+        }
+        Ok(())
+    }
+
+    fn with_requests(
+        mut self,
+        requests: impl IntoIterator<Item = BlobRangeRequest>,
+        selection: fn(Vec<u64>) -> ReadBlobsSelection,
+    ) -> Self {
+        let requests = requests.into_iter();
+        let (lower_bound, _) = requests.size_hint();
+        let mut rows = Vec::with_capacity(lower_bound);
+        let mut ranges = Vec::with_capacity(lower_bound);
+        for request in requests {
+            rows.push(request.row);
+            ranges.push(request.range);
+        }
+        self.inner.selection = selection(rows);
+        self.ranges = ranges;
+        self
+    }
+}
+
 /// One logical blob selected for planned reading.
 #[derive(Debug)]
 struct BlobEntry {
@@ -1658,14 +1940,38 @@ struct BlobEntry {
     file: BlobFile,
 }
 
+/// Selected rows together with the subset that has a readable blob payload.
+#[derive(Debug)]
+struct CollectedBlobSelection {
+    entries: Vec<BlobEntry>,
+    row_addresses: Vec<u64>,
+}
+
 /// Physical read input derived from one [`BlobEntry`].
 #[derive(Debug)]
 struct PlannedBlobRead {
     selection_index: usize,
     row_address: u64,
+    requested_range: Option<BlobReadRange>,
     physical_range: Range<u64>,
 }
 
+/// A slice of one disjoint physical range submitted to the file scheduler.
+#[derive(Debug)]
+struct PlannedBlobReadSlice {
+    read_index: usize,
+    physical_range_index: usize,
+    relative_range: Range<u64>,
+}
+
+/// One physical read paired with the backing source used to batch it.
+#[derive(Debug)]
+struct SourcePlannedBlobRead {
+    source_key: BlobSourceKey,
+    source: Arc<BlobSource>,
+    read: PlannedBlobRead,
+}
+
 /// One per-source read plan emitted by `read_blobs`.
 #[derive(Debug)]
 struct BlobReadPlan {
@@ -1674,6 +1980,127 @@ struct BlobReadPlan {
     reads: Vec<PlannedBlobRead>,
 }
 
+/// A payload-bounded group of per-source plans.
+#[derive(Debug)]
+struct BlobReadBatch {
+    plans: Vec<BlobReadPlan>,
+    expected_selection_indices: VecDeque<usize>,
+}
+
+#[derive(Debug)]
+struct CoalescedRangeInsertion {
+    merged: Range<u64>,
+    replaced_starts: Vec<u64>,
+    replaced_bytes: u64,
+}
+
+impl CoalescedRangeInsertion {
+    fn additional_bytes(&self) -> Result<u64> {
+        let merged_bytes = self.merged.end - self.merged.start;
+        merged_bytes.checked_sub(self.replaced_bytes).ok_or_else(|| {
+            Error::internal(format!(
+                "Coalesced blob range {:?} was smaller than its replaced ranges totaling {} bytes",
+                self.merged, self.replaced_bytes
+            ))
+        })
+    }
+}
+
+/// Physical ranges already charged to one source in the current batch.
+#[derive(Debug, Default)]
+struct CoalescedPhysicalRanges {
+    ranges: BTreeMap<u64, u64>,
+}
+
+impl CoalescedPhysicalRanges {
+    fn plan_insertion(
+        &self,
+        range: &Range<u64>,
+        block_size: u64,
+    ) -> Result<Option<CoalescedRangeInsertion>> {
+        if range.is_empty() {
+            return Ok(None);
+        }
+
+        let scan_start = self
+            .ranges
+            .range(..=range.start)
+            .next_back()
+            .map(|(start, _)| *start)
+            .unwrap_or(range.start);
+        let mut merged = range.clone();
+        let mut replaced_starts = Vec::new();
+        let mut replaced_bytes = 0_u64;
+        for (&start, &end) in self.ranges.range(scan_start..) {
+            if end.saturating_add(block_size) < merged.start {
+                continue;
+            }
+            if start > merged.end.saturating_add(block_size) {
+                break;
+            }
+            merged.start = merged.start.min(start);
+            merged.end = merged.end.max(end);
+            replaced_starts.push(start);
+            replaced_bytes = replaced_bytes
+                .checked_add(end - start)
+                .ok_or_else(|| Error::internal("Coalesced blob range size overflow".to_string()))?;
+        }
+
+        Ok(Some(CoalescedRangeInsertion {
+            merged,
+            replaced_starts,
+            replaced_bytes,
+        }))
+    }
+
+    fn insert(&mut self, insertion: CoalescedRangeInsertion) {
+        for start in insertion.replaced_starts {
+            self.ranges.remove(&start);
+        }
+        self.ranges
+            .insert(insertion.merged.start, insertion.merged.end);
+    }
+}
+
+/// Exact scheduler-visible physical footprint of the current batch.
+#[derive(Debug, Default)]
+struct BlobReadBatchFootprint {
+    sources: HashMap<BlobSourceKey, CoalescedPhysicalRanges>,
+    physical_bytes: u64,
+}
+
+impl BlobReadBatchFootprint {
+    fn try_insert(
+        &mut self,
+        planned: &SourcePlannedBlobRead,
+        physical_buffer_size: u64,
+        allow_oversized: bool,
+    ) -> Result<bool> {
+        let remaining = physical_buffer_size.saturating_sub(self.physical_bytes);
+        let additional_bytes = {
+            let source_ranges = self.sources.entry(planned.source_key.clone()).or_default();
+            let insertion = source_ranges.plan_insertion(
+                &planned.read.physical_range,
+                planned.source.object_store.block_size() as u64,
+            )?;
+            let Some(insertion) = insertion else {
+                return Ok(true);
+            };
+            let additional_bytes = insertion.additional_bytes()?;
+            if !allow_oversized && additional_bytes > remaining {
+                return Ok(false);
+            }
+            source_ranges.insert(insertion);
+            additional_bytes
+        };
+        self.physical_bytes = self
+            .physical_bytes
+            .checked_add(additional_bytes)
+            .ok_or_else(|| Error::internal("Blob read batch size overflow".to_string()))?;
+        Ok(true)
+    }
+}
+
 /// Operation-scoped scheduler cache for one [`ReadBlobsBuilder`] execution.
 ///
 /// We reuse one [`ScanScheduler`] per object store during a single `read_blobs`
@@ -1715,109 +2142,506 @@ impl ReadBlobsExecution {
 struct IndexedReadBlob {
     selection_index: usize,
     row_address: u64,
+    requested_range: Option<BlobReadRange>,
     data: Bytes,
 }
 
 fn into_read_blob(blob: IndexedReadBlob) -> ReadBlob {
+    debug_assert!(
+        blob.requested_range.is_none(),
+        "whole-blob reads must not carry a requested range"
+    );
     ReadBlob {
         row_address: blob.row_address,
-        data: blob.data,
+        data: Some(blob.data),
     }
 }
 
-/// Group selected blobs by physical source and sort each group's ranges by
-/// physical offset before handing them to the file scheduler.
-fn plan_blob_read_plans(entries: Vec<BlobEntry>) -> Vec<BlobReadPlan> {
-    let mut plan_indices = HashMap::<BlobSourceKey, usize>::new();
-    let mut plans = Vec::<BlobReadPlan>::new();
+fn collect_null_read_blobs(
+    entries: &[BlobEntry],
+    row_addresses: Vec<u64>,
+) -> Result<(usize, Vec<(usize, ReadBlob)>)> {
+    collect_null_selection_values(entries, row_addresses, |_, row_address| {
+        Ok(ReadBlob {
+            row_address,
+            data: None,
+        })
+    })
+}
 
-    for entry in entries {
-        let source_key = BlobSourceKey::new(&entry.file.source);
-        let plan_index = if let Some(plan_index) = plan_indices.get(&source_key) {
-            *plan_index
-        } else {
-            let plan_index = plans.len();
-            plans.push(BlobReadPlan {
-                source_key: source_key.clone(),
-                source: entry.file.source.clone(),
-                reads: Vec::new(),
-            });
-            plan_indices.insert(source_key.clone(), plan_index);
-            plan_index
-        };
+fn collect_null_read_blob_ranges(
+    entries: &[BlobEntry],
+    row_addresses: Vec<u64>,
+    ranges: &[BlobReadRange],
+) -> Result<(usize, Vec<(usize, ReadBlobRange)>)> {
+    collect_null_selection_values(entries, row_addresses, |request_index, row_address| {
+        let range = ranges.get(request_index).copied().ok_or_else(|| {
+            Error::internal(format!("Missing blob range for request {}", request_index))
+        })?;
+        Ok(ReadBlobRange {
+            request_index,
+            row_address,
+            range,
+            data: None,
+        })
+    })
+}
 
-        plans[plan_index].reads.push(PlannedBlobRead {
-            selection_index: entry.selection_index,
-            row_address: entry.row_address,
-            physical_range: entry.file.position..(entry.file.position + entry.file.size),
-        });
+fn collect_null_selection_values<T>(
+    entries: &[BlobEntry],
+    row_addresses: Vec<u64>,
+    mut make_value: impl FnMut(usize, u64) -> Result<T>,
+) -> Result<(usize, Vec<(usize, T)>)> {
+    let selection_count = row_addresses.len();
+    let mut non_null = vec![false; selection_count];
+    for entry in entries {
+        let is_non_null = non_null.get_mut(entry.selection_index).ok_or_else(|| {
+            Error::internal(format!(
+                "Blob selection index {} exceeded selected row count {}",
+                entry.selection_index, selection_count
+            ))
+        })?;
+        if *is_non_null {
+            return Err(Error::internal(format!(
+                "Blob selection index {} was collected more than once",
+                entry.selection_index
+            )));
+        }
+        *is_non_null = true;
     }
 
-    plans.sort_by(|left, right| {
-        left.source_key
-            .store_prefix
-            .cmp(&right.source_key.store_prefix)
-            .then_with(|| left.source_key.path.cmp(&right.source_key.path))
-    });
+    let mut null_values = Vec::with_capacity(selection_count.saturating_sub(entries.len()));
+    for (selection_index, row_address) in row_addresses.into_iter().enumerate() {
+        if !non_null[selection_index] {
+            null_values.push((selection_index, make_value(selection_index, row_address)?));
+        }
+    }
+    Ok((selection_count, null_values))
+}
 
-    for plan in &mut plans {
-        plan.reads.sort_by(|left, right| {
-            left.physical_range
-                .start
-                .cmp(&right.physical_range.start)
-                .then_with(|| left.physical_range.end.cmp(&right.physical_range.end))
-                .then_with(|| left.selection_index.cmp(&right.selection_index))
-        });
+fn totalize_blob_selection_stream<T: Send + 'static>(
+    non_null_values: BoxStream<'static, Result<(usize, T)>>,
+    null_values: Vec<(usize, T)>,
+    selection_count: usize,
+    preserve_order: bool,
+) -> BoxStream<'static, Result<T>> {
+    if !preserve_order {
+        let null_values = stream::iter(null_values.into_iter().map(|(_, value)| Ok(value)));
+        return stream::select(null_values, non_null_values.map_ok(|(_, value)| value)).boxed();
     }
 
-    plans
-}
+    let mut non_null_values = non_null_values;
+    let mut next_selection_index = 0;
+    let mut ready = null_values.into_iter().collect::<BTreeMap<_, _>>();
+    stream::poll_fn(move |cx| {
+        loop {
+            if next_selection_index == selection_count {
+                return Poll::Ready(None);
+            }
 
-/// Execute one per-source blob read plan with a single scheduler submission.
-async fn execute_blob_read_plan(
-    task: BlobReadPlan,
-    execution: Arc<ReadBlobsExecution>,
-) -> Result<Vec<IndexedReadBlob>> {
-    let ranges = task
-        .reads
-        .iter()
-        .map(|read| read.physical_range.clone())
-        .collect::<Vec<_>>();
-    let scheduler = execution.scheduler_for(&task.source);
-    let file_scheduler = scheduler
-        .open_file(&task.source.path, &task.source.file_size)
-        .await?;
-    let priority = ranges[0].start;
-    let bytes = file_scheduler.submit_request(ranges, priority).await?;
+            if let Some(value) = ready.remove(&next_selection_index) {
+                next_selection_index += 1;
+                return Poll::Ready(Some(Ok(value)));
+            }
 
-    Ok(task
-        .reads
-        .into_iter()
+            match non_null_values.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok((selection_index, value)))) => {
+                    if selection_index >= selection_count {
+                        return Poll::Ready(Some(Err(Error::internal(format!(
+                            "Blob selection index {} exceeded selected row count {}",
+                            selection_index, selection_count
+                        )))));
+                    }
+                    if ready.insert(selection_index, value).is_some() {
+                        return Poll::Ready(Some(Err(Error::internal(format!(
+                            "Blob selection index {} was produced more than once",
+                            selection_index
+                        )))));
+                    }
+                }
+                Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err))),
+                Poll::Ready(None) => {
+                    return Poll::Ready(Some(Err(Error::internal(format!(
+                        "planned blob read stream completed before selection index {} was produced",
+                        next_selection_index
+                    )))));
+                }
+                Poll::Pending => return Poll::Pending,
+            }
+        }
+    })
+    .boxed()
+}
+
+fn into_read_blob_range(blob: IndexedReadBlob) -> Result<ReadBlobRange> {
+    let range = blob.requested_range.ok_or_else(|| {
+        Error::internal(format!(
+            "Blob range request {} completed without range metadata",
+            blob.selection_index
+        ))
+    })?;
+    Ok(ReadBlobRange {
+        request_index: blob.selection_index,
+        row_address: blob.row_address,
+        range,
+        data: Some(blob.data),
+    })
+}
+
+/// Split selected reads into request-order batches before grouping each batch
+/// by physical source. Each batch accounts for the physical spans that the file
+/// scheduler will read after coalescing nearby ranges.
+fn plan_blob_read_batches(
+    mut entries: Vec<BlobEntry>,
+    requested_ranges: Option<&[BlobReadRange]>,
+    physical_buffer_size: u64,
+) -> Result<Vec<BlobReadBatch>> {
+    debug_assert!(physical_buffer_size > 0);
+    entries.sort_by_key(|entry| entry.selection_index);
+    let planned_reads = plan_blob_reads(entries, requested_ranges)?;
+
+    let mut batches = Vec::new();
+    let mut current_reads = Vec::new();
+    let mut footprint = BlobReadBatchFootprint::default();
+    for planned in planned_reads {
+        if !footprint.try_insert(&planned, physical_buffer_size, current_reads.is_empty())? {
+            batches.push(into_blob_read_batch(std::mem::take(&mut current_reads)));
+            footprint = BlobReadBatchFootprint::default();
+            let inserted = footprint.try_insert(&planned, physical_buffer_size, true)?;
+            debug_assert!(inserted, "an empty batch must accept one physical read");
+        }
+        current_reads.push(planned);
+    }
+    if !current_reads.is_empty() {
+        batches.push(into_blob_read_batch(current_reads));
+    }
+
+    Ok(batches)
+}
+
+fn plan_blob_reads(
+    entries: Vec<BlobEntry>,
+    requested_ranges: Option<&[BlobReadRange]>,
+) -> Result<Vec<SourcePlannedBlobRead>> {
+    let mut planned_reads = Vec::with_capacity(entries.len());
+    for entry in entries {
+        let requested_range = requested_ranges
+            .map(|ranges| {
+                ranges.get(entry.selection_index).copied().ok_or_else(|| {
+                    Error::internal(format!(
+                        "Missing blob range for request {}",
+                        entry.selection_index
+                    ))
+                })
+            })
+            .transpose()?;
+        let logical_range = match requested_range {
+            Some(range) => range.checked_range(entry.selection_index)?,
+            None => 0..entry.file.size,
+        };
+        let physical_range = entry.file.read_phys_range(logical_range).map_err(|err| {
+            Error::invalid_input(format!(
+                "Blob range request {} for row address {} is invalid: {}",
+                entry.selection_index, entry.row_address, err
+            ))
+        })?;
+        let source_key = BlobSourceKey::new(&entry.file.source);
+        planned_reads.push(SourcePlannedBlobRead {
+            source_key,
+            source: entry.file.source.clone(),
+            read: PlannedBlobRead {
+                selection_index: entry.selection_index,
+                row_address: entry.row_address,
+                requested_range,
+                physical_range,
+            },
+        });
+    }
+
+    Ok(planned_reads)
+}
+
+fn into_blob_read_batch(planned_reads: Vec<SourcePlannedBlobRead>) -> BlobReadBatch {
+    let expected_selection_indices = planned_reads
+        .iter()
+        .map(|planned| planned.read.selection_index)
+        .collect();
+    BlobReadBatch {
+        plans: group_blob_read_plans(planned_reads),
+        expected_selection_indices,
+    }
+}
+
+/// Group selected blobs by physical source and sort each group's ranges by
+/// physical offset before handing them to the file scheduler.
+fn group_blob_read_plans(planned_reads: Vec<SourcePlannedBlobRead>) -> Vec<BlobReadPlan> {
+    let mut plan_indices = HashMap::<BlobSourceKey, usize>::new();
+    let mut plans = Vec::<BlobReadPlan>::new();
+
+    for planned in planned_reads {
+        let plan_index = if let Some(plan_index) = plan_indices.get(&planned.source_key) {
+            *plan_index
+        } else {
+            let plan_index = plans.len();
+            plans.push(BlobReadPlan {
+                source_key: planned.source_key.clone(),
+                source: planned.source,
+                reads: Vec::new(),
+            });
+            plan_indices.insert(planned.source_key, plan_index);
+            plan_index
+        };
+        plans[plan_index].reads.push(planned.read);
+    }
+
+    plans.sort_by(|left, right| {
+        left.source_key
+            .store_prefix
+            .cmp(&right.source_key.store_prefix)
+            .then_with(|| left.source_key.path.cmp(&right.source_key.path))
+    });
+
+    for plan in &mut plans {
+        plan.reads.sort_by(|left, right| {
+            left.physical_range
+                .start
+                .cmp(&right.physical_range.start)
+                .then_with(|| left.physical_range.end.cmp(&right.physical_range.end))
+                .then_with(|| left.selection_index.cmp(&right.selection_index))
+        });
+    }
+
+    plans
+}
+
+fn plan_blob_read_plans(
+    entries: Vec<BlobEntry>,
+    requested_ranges: Option<&[BlobReadRange]>,
+) -> Result<Vec<BlobReadPlan>> {
+    Ok(group_blob_read_plans(plan_blob_reads(
+        entries,
+        requested_ranges,
+    )?))
+}
+
+/// Merge overlapping physical reads before submitting them to [`FileScheduler`].
+///
+/// `FileScheduler` can safely coalesce and split disjoint ranges. Original
+/// overlapping ranges need to be mapped onto their union first so a split does
+/// not advance past the start of a later nested range while reconstructing the
+/// caller's buffers.
+fn plan_disjoint_blob_reads(
+    reads: &[PlannedBlobRead],
+) -> (Vec<Range<u64>>, Vec<PlannedBlobReadSlice>) {
+    let mut non_empty_ranges = reads
+        .iter()
+        .enumerate()
+        .filter(|(_, read)| !read.physical_range.is_empty())
+        .map(|(read_index, read)| (read_index, read.physical_range.clone()))
+        .collect::<Vec<_>>();
+    non_empty_ranges.sort_by_key(|(read_index, range)| (range.start, range.end, *read_index));
+
+    let mut physical_ranges = Vec::<Range<u64>>::with_capacity(non_empty_ranges.len());
+    let mut slices = Vec::with_capacity(non_empty_ranges.len());
+    for (read_index, range) in non_empty_ranges {
+        let physical_range_index = match physical_ranges.last_mut() {
+            Some(physical_range) if range.start <= physical_range.end => {
+                physical_range.end = physical_range.end.max(range.end);
+                physical_ranges.len() - 1
+            }
+            _ => {
+                physical_ranges.push(range.clone());
+                physical_ranges.len() - 1
+            }
+        };
+        let physical_start = physical_ranges[physical_range_index].start;
+        slices.push(PlannedBlobReadSlice {
+            read_index,
+            physical_range_index,
+            relative_range: range.start - physical_start..range.end - physical_start,
+        });
+    }
+
+    (physical_ranges, slices)
+}
+
+/// Execute one per-source blob read plan with a single scheduler submission.
+async fn execute_blob_read_plan(
+    task: BlobReadPlan,
+    execution: Arc<ReadBlobsExecution>,
+) -> Result<Vec<IndexedReadBlob>> {
+    let (physical_ranges, slices) = plan_disjoint_blob_reads(&task.reads);
+    let mut bytes = vec![Bytes::new(); task.reads.len()];
+    if let Some(first_range) = physical_ranges.first() {
+        let scheduler = execution.scheduler_for(&task.source);
+        let file_scheduler = scheduler
+            .open_file(&task.source.path, &task.source.file_size)
+            .await?;
+        let priority = first_range.start;
+        let physical_range_count = physical_ranges.len();
+        let returned = file_scheduler
+            .submit_request(physical_ranges, priority)
+            .await?;
+        if returned.len() != physical_range_count {
+            return Err(Error::internal(format!(
+                "Blob read scheduler returned {} ranges for {} disjoint physical ranges from {}",
+                returned.len(),
+                physical_range_count,
+                task.source.path
+            )));
+        }
+        for slice in slices {
+            let start = usize::try_from(slice.relative_range.start).map_err(|_| {
+                Error::internal(format!(
+                    "Blob read slice start {} does not fit into usize for {}",
+                    slice.relative_range.start, task.source.path
+                ))
+            })?;
+            let end = usize::try_from(slice.relative_range.end).map_err(|_| {
+                Error::internal(format!(
+                    "Blob read slice end {} does not fit into usize for {}",
+                    slice.relative_range.end, task.source.path
+                ))
+            })?;
+            let data = &returned[slice.physical_range_index];
+            if end > data.len() {
+                return Err(Error::internal(format!(
+                    "Blob read slice {:?} exceeds the {} bytes returned for physical range {} from {}",
+                    slice.relative_range,
+                    data.len(),
+                    slice.physical_range_index,
+                    task.source.path
+                )));
+            }
+            bytes[slice.read_index] = data.slice(start..end);
+        }
+    }
+
+    Ok(task
+        .reads
+        .into_iter()
         .zip(bytes)
         .map(|(read, data)| IndexedReadBlob {
             selection_index: read.selection_index,
             row_address: read.row_address,
+            requested_range: read.requested_range,
             data,
         })
         .collect())
 }
 
+fn execute_blob_read_batches_stream(
+    batches: Vec<BlobReadBatch>,
+    execution: Arc<ReadBlobsExecution>,
+    io_parallelism: usize,
+    preserve_order: bool,
+) -> BoxStream<'static, Result<IndexedReadBlob>> {
+    let streams = batches.into_iter().map(move |batch| {
+        execute_blob_read_plans_stream(
+            batch.plans,
+            batch.expected_selection_indices,
+            execution.clone(),
+            io_parallelism,
+            preserve_order,
+        )
+    });
+    stream::iter(streams).flatten().boxed()
+}
+
+fn execute_blob_read_plans_stream(
+    plans: Vec<BlobReadPlan>,
+    expected_selection_indices: VecDeque<usize>,
+    execution: Arc<ReadBlobsExecution>,
+    io_parallelism: usize,
+    preserve_order: bool,
+) -> BoxStream<'static, Result<IndexedReadBlob>> {
+    if plans.is_empty() {
+        return stream::empty().boxed();
+    }
+
+    let plan_stream = stream::iter(plans.into_iter().map(move |plan| {
+        let execution = execution.clone();
+        execute_blob_read_plan(plan, execution)
+    }))
+    .buffer_unordered(io_parallelism.max(1));
+
+    if !preserve_order {
+        return plan_stream
+            .map_ok(|blobs| stream::iter(blobs.into_iter().map(Ok)))
+            .try_flatten()
+            .boxed();
+    }
+
+    let mut plan_stream = plan_stream.boxed();
+    let mut expected_selection_indices = expected_selection_indices;
+    let mut ready = BTreeMap::<usize, IndexedReadBlob>::new();
+    stream::poll_fn(move |cx| {
+        loop {
+            let Some(next_selection_index) = expected_selection_indices.front().copied() else {
+                return Poll::Ready(None);
+            };
+
+            if let Some(blob) = ready.remove(&next_selection_index) {
+                expected_selection_indices.pop_front();
+                return Poll::Ready(Some(Ok(blob)));
+            }
+
+            match plan_stream.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok(blobs))) => {
+                    for blob in blobs {
+                        ready.insert(blob.selection_index, blob);
+                    }
+                }
+                Poll::Ready(Some(Err(err))) => return Poll::Ready(Some(Err(err))),
+                Poll::Ready(None) => {
+                    let err = Error::internal(format!(
+                        "planned blob read stream completed before selection index {} was produced",
+                        next_selection_index
+                    ));
+                    return Poll::Ready(Some(Err(err)));
+                }
+                Poll::Pending => return Poll::Pending,
+            }
+        }
+    })
+    .boxed()
+}
+
+async fn execute_blob_entries(
+    entries: Vec<BlobEntry>,
+    io_parallelism: usize,
+    io_buffer_size_bytes: Option<u64>,
+) -> Result<Vec<IndexedReadBlob>> {
+    let plans = plan_blob_read_plans(entries, None)?;
+    if plans.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let execution = Arc::new(ReadBlobsExecution::new(io_buffer_size_bytes));
+    let batches = stream::iter(plans.into_iter().map(move |plan| {
+        let execution = execution.clone();
+        execute_blob_read_plan(plan, execution)
+    }))
+    .buffer_unordered(io_parallelism.max(1))
+    .try_collect::<Vec<_>>()
+    .await?;
+    Ok(batches.into_iter().flatten().collect())
+}
+
 pub(super) async fn take_blobs(
     dataset: &Arc<Dataset>,
     row_ids: &[u64],
     column: &str,
-) -> Result<Vec<BlobFile>> {
+) -> Result<Vec<Option<BlobFile>>> {
     let blob_field_id = validate_blob_column(dataset, column)?;
-    Ok(collect_blob_entries_for_selection(
+    let collected = collect_blob_selection_for_selection(
         dataset,
         blob_field_id,
         column,
         &ReadBlobsSelection::RowIds(row_ids.to_vec()),
     )
-    .await?
-    .into_iter()
-    .map(|entry| entry.file)
-    .collect())
+    .await?;
+    into_optional_blob_files(collected)
 }
 
 /// Take [BlobFile] by row addresses.
@@ -1831,18 +2655,38 @@ pub async fn take_blobs_by_addresses(
     dataset: &Arc<Dataset>,
     row_addrs: &[u64],
     column: &str,
-) -> Result<Vec<BlobFile>> {
+) -> Result<Vec<Option<BlobFile>>> {
     let blob_field_id = validate_blob_column(dataset, column)?;
-    Ok(collect_blob_entries_for_selection(
+    let collected = collect_blob_selection_for_selection(
         dataset,
         blob_field_id,
         column,
         &ReadBlobsSelection::RowAddresses(row_addrs.to_vec()),
     )
-    .await?
-    .into_iter()
-    .map(|entry| entry.file)
-    .collect())
+    .await?;
+    into_optional_blob_files(collected)
+}
+
+fn into_optional_blob_files(collected: CollectedBlobSelection) -> Result<Vec<Option<BlobFile>>> {
+    let selection_count = collected.row_addresses.len();
+    let mut files = std::iter::repeat_with(|| None)
+        .take(selection_count)
+        .collect::<Vec<_>>();
+    for entry in collected.entries {
+        let slot = files.get_mut(entry.selection_index).ok_or_else(|| {
+            Error::internal(format!(
+                "Blob selection index {} exceeded selected row count {}",
+                entry.selection_index, selection_count
+            ))
+        })?;
+        if slot.replace(entry.file).is_some() {
+            return Err(Error::internal(format!(
+                "Blob selection index {} was collected more than once",
+                entry.selection_index
+            )));
+        }
+    }
+    Ok(files)
 }
 
 /// Validate that `column` exists and is a blob column, returning its field id.
@@ -1888,14 +2732,13 @@ async fn take_blob_descriptions_by_row_addresses(
         .await
 }
 
-/// Resolve a caller selection into [`BlobEntry`] values that share `BlobSource`
-/// instances by physical backing object.
-async fn collect_blob_entries_for_selection(
+/// Resolve every selected row address and the non-null blob entries among them.
+async fn collect_blob_selection_for_selection(
     dataset: &Arc<Dataset>,
     blob_field_id: u32,
     column: &str,
     selection: &ReadBlobsSelection,
-) -> Result<Vec<BlobEntry>> {
+) -> Result<CollectedBlobSelection> {
     let description_and_addr = match selection {
         ReadBlobsSelection::None => {
             return Err(Error::invalid_input(
@@ -1917,34 +2760,83 @@ async fn collect_blob_entries_for_selection(
     };
 
     if description_and_addr.num_rows() == 0 {
-        return Ok(Vec::new());
+        return Ok(CollectedBlobSelection {
+            entries: Vec::new(),
+            row_addresses: Vec::new(),
+        });
     }
 
     let descriptions = leaf_descriptor_struct(&description_and_addr, column)?;
     let row_addrs = description_and_addr.column(1).as_primitive::<UInt64Type>();
+    let row_addresses = row_addrs.values().to_vec();
 
-    match blob_version_from_descriptions(descriptions)? {
+    let entries = match blob_version_from_descriptions(descriptions)? {
         BlobVersion::V1 => collect_blob_entries_v1(dataset, blob_field_id, descriptions, row_addrs),
         BlobVersion::V2 => {
             collect_blob_entries_v2(dataset, blob_field_id, descriptions, row_addrs).await
         }
-    }
+    }?;
+    Ok(CollectedBlobSelection {
+        entries,
+        row_addresses,
+    })
 }
 
 /// Walk into the descriptor `RecordBatch` at `column` and return the leaf
 /// descriptor `StructArray`, descending through nested struct children for
 /// dotted paths.
 fn leaf_descriptor_struct<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a StructArray> {
+    let current = leaf_descriptor_array(batch, column)?;
+    current
+        .as_any()
+        .downcast_ref::<StructArray>()
+        .ok_or_else(|| {
+            Error::invalid_input_source(
+                format!(
+                    "Blob column '{}' expected descriptor struct but got {}",
+                    column,
+                    current.data_type()
+                )
+                .into(),
+            )
+        })
+}
+
+fn leaf_descriptor_array<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a dyn Array> {
     let path = parse_field_path(column)?;
-    let mut current = batch
+    let mut current: &dyn Array = batch
         .column_by_name(&path[0])
-        .expect("validate_blob_column ensured column exists")
-        .as_struct();
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Blob column '{}' was not found in descriptor batch",
+                column
+            ))
+        })?
+        .as_ref();
     for segment in &path[1..] {
-        current = current
+        let struct_array = current
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "Blob column path '{}' expected struct before segment '{}' but got {}",
+                        column,
+                        segment,
+                        current.data_type()
+                    )
+                    .into(),
+                )
+            })?;
+        current = struct_array
             .column_by_name(segment)
-            .expect("validate_blob_column ensured all path segments exist")
-            .as_struct();
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Blob column path '{}' missing segment '{}'",
+                    column, segment
+                ))
+            })?
+            .as_ref();
     }
     Ok(current)
 }
@@ -1970,6 +2862,32 @@ fn blob_version_from_descriptions(descriptions: &StructArray) -> Result<BlobVers
     .into()))
 }
 
+struct BlobV2DescriptorColumns<'a> {
+    descriptions: &'a StructArray,
+    kinds: &'a arrow::array::PrimitiveArray<UInt8Type>,
+    positions: &'a arrow::array::PrimitiveArray<UInt64Type>,
+    sizes: &'a arrow::array::PrimitiveArray<UInt64Type>,
+    blob_ids: &'a arrow::array::PrimitiveArray<UInt32Type>,
+    blob_uris: &'a arrow::array::GenericStringArray<i32>,
+}
+
+impl<'a> BlobV2DescriptorColumns<'a> {
+    fn new(descriptions: &'a StructArray) -> Self {
+        Self {
+            descriptions,
+            kinds: descriptions.column(0).as_primitive::<UInt8Type>(),
+            positions: descriptions.column(1).as_primitive::<UInt64Type>(),
+            sizes: descriptions.column(2).as_primitive::<UInt64Type>(),
+            blob_ids: descriptions.column(3).as_primitive::<UInt32Type>(),
+            blob_uris: descriptions.column(4).as_string::<i32>(),
+        }
+    }
+
+    fn is_null_blob(&self, idx: usize) -> bool {
+        self.descriptions.is_null(idx) || self.kinds.is_null(idx)
+    }
+}
+
 /// Convert blob v1 descriptors into logical blob entries.
 fn collect_blob_entries_v1(
     dataset: &Arc<Dataset>,
@@ -1989,6 +2907,11 @@ fn collect_blob_entries_v1(
         .filter_map(|(selection_index, ((row_addr, position), size))| {
             let position = position?;
             let size = size?;
+            // V1 encodes valid empty blobs as (0, 0) and smuggles null
+            // repetition/definition levels through a non-zero zero-sized position.
+            if descriptions.is_null(selection_index) || (size == 0 && position != 0) {
+                return None;
+            }
             Some((selection_index, *row_addr, position, size))
         })
         .map(|(selection_index, row_addr, position, size)| {
@@ -2032,185 +2955,601 @@ async fn collect_blob_entries_v2(
     descriptions: &StructArray,
     row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
 ) -> Result<Vec<BlobEntry>> {
-    let kinds = descriptions.column(0).as_primitive::<UInt8Type>();
-    let positions = descriptions.column(1).as_primitive::<UInt64Type>();
-    let sizes = descriptions.column(2).as_primitive::<UInt64Type>();
-    let blob_ids = descriptions.column(3).as_primitive::<UInt32Type>();
-    let blob_uris = descriptions.column(4).as_string::<i32>();
-
+    let columns = BlobV2DescriptorColumns::new(descriptions);
     let mut files = Vec::with_capacity(row_addrs.len());
-    let mut fragment_cache = HashMap::<u32, BlobReadLocation>::new();
-    let mut store_cache = HashMap::<u32, Arc<ObjectStore>>::new();
-    let mut external_base_path_cache = HashMap::<u32, Path>::new();
-    let mut source_cache = HashMap::<BlobSourceKey, Arc<BlobSource>>::new();
+    let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id);
     for (selection_index, row_addr) in row_addrs.values().iter().enumerate() {
-        let idx = selection_index;
-        let kind = BlobKind::try_from(kinds.value(idx))?;
-
-        // Struct is non-nullable; null rows are encoded as inline with zero position/size and empty uri
-        if matches!(kind, BlobKind::Inline) && positions.value(idx) == 0 && sizes.value(idx) == 0 {
-            continue;
-        }
-
-        match kind {
-            BlobKind::Inline => {
-                let position = positions.value(idx);
-                let size = sizes.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let source = shared_blob_source(
-                    &mut source_cache,
-                    location.object_store,
-                    &location.data_file_path,
-                );
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, position, size, BlobKind::Inline, None),
-                });
-            }
-            BlobKind::Dedicated => {
-                let blob_id = blob_ids.value(idx);
-                let size = sizes.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None),
-                });
-            }
-            BlobKind::Packed => {
-                let blob_id = blob_ids.value(idx);
-                let size = sizes.value(idx);
-                let position = positions.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, position, size, BlobKind::Packed, None),
-                });
-            }
-            BlobKind::External => {
-                let uri_or_path = blob_uris.value(idx).to_string();
-                let position = positions.value(idx);
-                let size = sizes.value(idx);
-                let base_id = blob_ids.value(idx);
-                let (object_store, path) = if base_id == 0 {
-                    let registry = dataset.session.store_registry();
-                    let params = dataset
-                        .store_params
-                        .as_ref()
-                        .map(|p| Arc::new((**p).clone()))
-                        .unwrap_or_else(|| Arc::new(ObjectStoreParams::default()));
-                    ObjectStore::from_uri_and_params(registry, &uri_or_path, &params).await?
-                } else {
-                    let object_store = if let Some(store) = store_cache.get(&base_id) {
-                        store.clone()
-                    } else {
-                        let store = dataset.object_store(Some(base_id)).await?;
-                        store_cache.insert(base_id, store.clone());
-                        store
-                    };
-                    let base_root = if let Some(path) = external_base_path_cache.get(&base_id) {
-                        path.clone()
-                    } else {
-                        let base = dataset.manifest.base_paths.get(&base_id).ok_or_else(|| {
-                            Error::invalid_input(format!(
-                                "External blob references unknown base_id {}",
-                                base_id
-                            ))
-                        })?;
-                        let path = base.extract_path(dataset.session.store_registry())?;
-                        external_base_path_cache.insert(base_id, path.clone());
-                        path
-                    };
-                    let path = join_base_and_relative_path(&base_root, &uri_or_path)?;
-                    (object_store, path)
-                };
-                let size = if size > 0 {
-                    size
-                } else {
-                    object_store.size(&path).await?
-                };
-                let source = shared_blob_source(&mut source_cache, object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(
-                        source,
-                        position,
-                        size,
-                        BlobKind::External,
-                        Some(uri_or_path),
-                    ),
-                });
-            }
+        if let Some(entry) = read_context
+            .collect_entry(&columns, selection_index, selection_index, *row_addr)
+            .await?
+        {
+            files.push(entry);
         }
     }
 
     Ok(files)
 }
 
-fn normalize_external_absolute_uri(uri: &str) -> Result<String> {
-    let url = Url::parse(uri).map_err(|_| {
-        Error::invalid_input(format!(
-            "External URI '{}' is outside registered external bases and is not a valid absolute URI",
-            uri
-        ))
-    })?;
-    Ok(url.to_string())
+fn is_blob_v2_binary_view(field: &LanceField) -> bool {
+    field.is_blob_v2() && matches!(field.data_type(), ArrowDataType::LargeBinary)
 }
 
-fn join_base_and_relative_path(base: &Path, relative_path: &str) -> Result<Path> {
-    let relative = Path::parse(relative_path).map_err(|e| {
-        Error::invalid_input(format!(
-            "Invalid relative external blob path '{}': {}",
-            relative_path, e
-        ))
-    })?;
-    Ok(Path::from_iter(base.parts().chain(relative.parts())))
+fn public_blob_v2_binary_output_field(mut field: LanceField) -> LanceField {
+    if is_blob_v2_binary_view(&field) {
+        field.metadata.remove(ARROW_EXT_NAME_KEY);
+    }
+    field.children = field
+        .children
+        .into_iter()
+        .map(public_blob_v2_binary_output_field)
+        .collect();
+    field
 }
 
-/// Resolve the physical read location for a blob row in a base-aware way.
+/// Return the public Arrow-facing schema for a blob v2 binary scan.
 ///
-/// Given a `row_addr`, this helper locates the owning fragment and the blob field's
-/// data file, then returns the concrete object store and paths needed to read blob
-/// bytes correctly under multi-base datasets.
+/// Scan planning uses a blob v2 extension marker on `LargeBinary` leaves to
+/// identify payloads that need descriptor-based materialization. This helper
+/// removes that internal marker before the schema is exposed to callers.
+pub fn public_blob_v2_binary_output_schema(schema: &Schema) -> Schema {
+    Schema {
+        fields: schema
+            .fields
+            .iter()
+            .cloned()
+            .map(public_blob_v2_binary_output_field)
+            .collect(),
+        metadata: schema.metadata.clone(),
+    }
+}
+
+fn field_has_blob_v2_binary_view(field: &LanceField) -> bool {
+    is_blob_v2_binary_view(field) || field.children.iter().any(field_has_blob_v2_binary_view)
+}
+
+/// Return true if the schema contains a blob v2 leaf in binary payload view.
 ///
-/// It uses two caller-provided caches:
-/// - `fragment_cache` memoizes per-fragment path metadata (`data_file_dir`,
-///   `data_file_path`, and `data_file_key`) plus the resolved store.
-/// - `store_cache` memoizes `base_id -> ObjectStore` so multiple fragments that
-///   share the same base do not repeat async store resolution.
-async fn resolve_blob_read_location(
-    dataset: &Arc<Dataset>,
-    blob_field_id: u32,
-    row_addr: u64,
-    fragment_cache: &mut HashMap<u32, BlobReadLocation>,
+/// This detects the internal `LargeBinary` view created by
+/// [`BlobHandling::AllBinary`](lance_core::datatypes::BlobHandling::AllBinary)
+/// or selective binary blob handling.
+pub fn schema_has_blob_v2_binary_view(schema: &Schema) -> bool {
+    schema.fields.iter().any(field_has_blob_v2_binary_view)
+}
+
+fn blob_v2_descriptor_field(mut field: LanceField) -> LanceField {
+    if is_blob_v2_binary_view(&field) {
+        field.unloaded_mut();
+        return field;
+    }
+
+    field.children = field
+        .children
+        .into_iter()
+        .map(blob_v2_descriptor_field)
+        .collect();
+    field
+}
+
+/// Convert blob v2 binary-view leaves back to descriptor-view leaves.
+///
+/// Readers use this schema to fetch stored blob descriptors first. The scan
+/// layer then materializes those descriptors into the caller's binary payload
+/// view after row addresses are available.
+pub fn blob_v2_descriptor_schema(schema: &Schema) -> Schema {
+    Schema {
+        fields: schema
+            .fields
+            .iter()
+            .cloned()
+            .map(blob_v2_descriptor_field)
+            .collect(),
+        metadata: schema.metadata.clone(),
+    }
+}
+
+/// Materialize blob v2 descriptor arrays in a decoded batch into binary arrays.
+///
+/// The input batch must include `_rowaddr`, which is used to resolve packed,
+/// dedicated, inline, and external blob payload locations. `output_schema`
+/// defines the exact returned columns, including requested system columns, with
+/// blob v2 binary leaves exposed as plain `LargeBinary` fields.
+pub async fn materialize_blob_v2_binary_batch(
+    dataset: &Arc<Dataset>,
+    output_schema: &Schema,
+    batch: RecordBatch,
+) -> Result<RecordBatch> {
+    let row_addr_idx = batch
+        .schema()
+        .column_with_name(ROW_ADDR)
+        .ok_or_else(|| {
+            Error::internal(format!(
+                "_rowaddr column missing from blob v2 binary scan batch, columns: {:?}",
+                batch
+                    .schema()
+                    .fields()
+                    .iter()
+                    .map(|field| field.name())
+                    .collect::<Vec<_>>()
+            ))
+        })?
+        .0;
+    let row_addrs = batch
+        .column(row_addr_idx)
+        .as_primitive::<UInt64Type>()
+        .values()
+        .iter()
+        .copied()
+        .collect::<Vec<_>>();
+    let row_addrs: Arc<[u64]> = row_addrs.into();
+
+    let mut columns = Vec::with_capacity(output_schema.fields.len());
+    let mut fields = Vec::with_capacity(output_schema.fields.len());
+
+    for field in &output_schema.fields {
+        let input = batch
+            .column_by_name(&field.name)
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 binary scan batch missing projected column '{}'",
+                    field.name
+                ))
+            })?
+            .clone();
+        let materialized =
+            materialize_blob_v2_binary_array(dataset, field, input, row_addrs.clone()).await?;
+        columns.push(materialized);
+        let output_field = public_blob_v2_binary_output_field(field.clone());
+        fields.push(ArrowField::from(&output_field));
+    }
+
+    Ok(RecordBatch::try_new(
+        Arc::new(ArrowSchema::new_with_metadata(
+            fields,
+            batch.schema().metadata().clone(),
+        )),
+        columns,
+    )?)
+}
+
+fn materialize_blob_v2_binary_array<'a>(
+    dataset: &'a Arc<Dataset>,
+    field: &'a LanceField,
+    array: ArrayRef,
+    row_addrs: Arc<[u64]>,
+) -> BoxFuture<'a, Result<ArrayRef>> {
+    async move {
+        if is_blob_v2_binary_view(field) {
+            let descriptions = array.as_struct();
+            return materialize_blob_v2_descriptors(
+                dataset,
+                field.id as u32,
+                descriptions,
+                row_addrs.as_ref(),
+            )
+            .await;
+        }
+
+        match field.data_type() {
+            ArrowDataType::Struct(_) => {
+                let struct_array = array.as_struct();
+                let mut children = Vec::with_capacity(field.children.len());
+                for (child_field, child_array) in
+                    field.children.iter().zip(struct_array.columns().iter())
+                {
+                    children.push(
+                        materialize_blob_v2_binary_array(
+                            dataset,
+                            child_field,
+                            child_array.clone(),
+                            row_addrs.clone(),
+                        )
+                        .await?,
+                    );
+                }
+                let public_field = public_blob_v2_binary_output_field(field.clone());
+                let ArrowDataType::Struct(fields) = public_field.data_type() else {
+                    unreachable!("public output field preserved struct type")
+                };
+                Ok(Arc::new(StructArray::try_new(
+                    fields,
+                    children,
+                    struct_array.nulls().cloned(),
+                )?) as ArrayRef)
+            }
+            ArrowDataType::List(_) => {
+                let list_array = array.as_list::<i32>();
+                materialize_blob_v2_list_array::<i32>(dataset, field, list_array, row_addrs).await
+            }
+            ArrowDataType::LargeList(_) => {
+                let list_array = array.as_list::<i64>();
+                materialize_blob_v2_list_array::<i64>(dataset, field, list_array, row_addrs).await
+            }
+            _ => Ok(array),
+        }
+    }
+    .boxed()
+}
+
+async fn materialize_blob_v2_list_array<O: OffsetSizeTrait>(
+    dataset: &Arc<Dataset>,
+    field: &LanceField,
+    list_array: &GenericListArray<O>,
+    row_addrs: Arc<[u64]>,
+) -> Result<ArrayRef> {
+    let offsets = list_array.value_offsets();
+    let values_start = offsets[0].as_usize();
+    let values_end = offsets[list_array.len()].as_usize();
+    if values_end < values_start {
+        return Err(Error::internal(format!(
+            "List field '{}' has invalid offsets while materializing blob v2 binary scan",
+            field.name
+        )));
+    }
+
+    let values_len = values_end - values_start;
+    let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1);
+    normalized_offsets.push(O::usize_as(0));
+    let mut child_row_addrs = Vec::with_capacity(values_len);
+    for row_idx in 0..list_array.len() {
+        let start = offsets[row_idx].as_usize();
+        let end = offsets[row_idx + 1].as_usize();
+        if end < start {
+            return Err(Error::internal(format!(
+                "List field '{}' has decreasing offsets while materializing blob v2 binary scan",
+                field.name
+            )));
+        }
+        let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| {
+            Error::internal(format!(
+                "List field '{}' row address count {} did not match row count {}",
+                field.name,
+                row_addrs.len(),
+                list_array.len()
+            ))
+        })?;
+        for _ in start..end {
+            child_row_addrs.push(row_addr);
+        }
+        normalized_offsets.push(O::usize_as(end - values_start));
+    }
+    let child_row_addrs: Arc<[u64]> = child_row_addrs.into();
+    let child = field.children.first().ok_or_else(|| {
+        Error::internal(format!(
+            "List field '{}' missing child while materializing blob v2 binary scan",
+            field.name
+        ))
+    })?;
+    let values = list_array.values().slice(values_start, values_len);
+    let values = materialize_blob_v2_binary_array(dataset, child, values, child_row_addrs).await?;
+    let child_field = public_blob_v2_binary_output_field(child.clone());
+    let list_array = GenericListArray::<O>::try_new(
+        Arc::new(ArrowField::from(&child_field)),
+        OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)),
+        values,
+        list_array.nulls().cloned(),
+    )?;
+    Ok(Arc::new(list_array))
+}
+
+async fn materialize_blob_v2_descriptors(
+    dataset: &Arc<Dataset>,
+    blob_field_id: u32,
+    descriptions: &StructArray,
+    row_addrs: &[u64],
+) -> Result<ArrayRef> {
+    if descriptions.len() != row_addrs.len() {
+        return Err(Error::internal(format!(
+            "blob v2 descriptor count {} did not match row address count {}",
+            descriptions.len(),
+            row_addrs.len()
+        )));
+    }
+    match blob_version_from_descriptions(descriptions)? {
+        BlobVersion::V1 => {
+            return Err(Error::not_supported(
+                "Blob v2 binary materialization received a legacy blob descriptor".to_string(),
+            ));
+        }
+        BlobVersion::V2 => {}
+    }
+
+    let columns = BlobV2DescriptorColumns::new(descriptions);
+    let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id);
+    let mut entries = Vec::with_capacity(descriptions.len());
+    let mut payloads = vec![None; descriptions.len()];
+
+    for (idx, row_addr) in row_addrs.iter().copied().enumerate() {
+        if descriptions.is_null(idx) || columns.kinds.is_null(idx) {
+            continue;
+        }
+
+        let kind = BlobKind::try_from(columns.kinds.value(idx))?;
+        if matches!(kind, BlobKind::Inline)
+            && columns.positions.value(idx) == 0
+            && columns.sizes.value(idx) == 0
+        {
+            payloads[idx] = Some(Bytes::new());
+            continue;
+        }
+
+        let entry = read_context
+            .collect_entry(&columns, idx, idx, row_addr)
+            .await?
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 descriptor at index {idx} unexpectedly resolved to null"
+                ))
+            })?;
+        entries.push(entry);
+    }
+
+    let blobs = execute_blob_entries(entries, dataset.object_store.io_parallelism(), None).await?;
+    for blob in blobs {
+        let payload = payloads.get_mut(blob.selection_index).ok_or_else(|| {
+            Error::internal(format!(
+                "blob result selection index {} exceeded descriptor count {}",
+                blob.selection_index,
+                descriptions.len()
+            ))
+        })?;
+        if payload.replace(blob.data).is_some() {
+            return Err(Error::internal(format!(
+                "blob result selection index {} was produced more than once",
+                blob.selection_index
+            )));
+        }
+    }
+
+    let payload_capacity = payloads.iter().flatten().map(Bytes::len).sum::<usize>();
+    let mut builder = LargeBinaryBuilder::with_capacity(descriptions.len(), payload_capacity);
+    for (idx, payload) in payloads.into_iter().enumerate() {
+        if descriptions.is_null(idx) || columns.kinds.is_null(idx) {
+            builder.append_null();
+        } else {
+            let payload = payload.ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 descriptor at index {idx} did not produce a payload"
+                ))
+            })?;
+            builder.append_value(payload);
+        }
+    }
+
+    Ok(Arc::new(builder.finish()))
+}
+
+struct BlobV2ReadContext<'a> {
+    dataset: &'a Arc<Dataset>,
+    blob_field_id: u32,
+    fragment_cache: HashMap<u32, BlobReadLocation>,
+    store_cache: HashMap<u32, Arc<ObjectStore>>,
+    external_base_path_cache: HashMap<u32, Path>,
+    source_cache: HashMap<BlobSourceKey, Arc<BlobSource>>,
+}
+
+impl<'a> BlobV2ReadContext<'a> {
+    fn new(dataset: &'a Arc<Dataset>, blob_field_id: u32) -> Self {
+        Self {
+            dataset,
+            blob_field_id,
+            fragment_cache: HashMap::new(),
+            store_cache: HashMap::new(),
+            external_base_path_cache: HashMap::new(),
+            source_cache: HashMap::new(),
+        }
+    }
+
+    async fn collect_entry(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<Option<BlobEntry>> {
+        if columns.is_null_blob(idx) {
+            return Ok(None);
+        }
+
+        let kind = BlobKind::try_from(columns.kinds.value(idx))?;
+        let entry = match kind {
+            BlobKind::Inline => {
+                self.collect_inline(columns, idx, selection_index, row_addr)
+                    .await?
+            }
+            BlobKind::Dedicated => {
+                self.collect_dedicated(columns, idx, selection_index, row_addr)
+                    .await?
+            }
+            BlobKind::Packed => {
+                self.collect_packed(columns, idx, selection_index, row_addr)
+                    .await?
+            }
+            BlobKind::External => {
+                self.collect_external(columns, idx, selection_index, row_addr)
+                    .await?
+            }
+        };
+
+        Ok(Some(entry))
+    }
+
+    async fn blob_read_location(&mut self, row_addr: u64) -> Result<BlobReadLocation> {
+        resolve_blob_read_location(
+            self.dataset,
+            self.blob_field_id,
+            row_addr,
+            &mut self.fragment_cache,
+            &mut self.store_cache,
+        )
+        .await
+    }
+
+    async fn collect_inline(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let position = columns.positions.value(idx);
+        let size = columns.sizes.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let source = shared_blob_source(
+            &mut self.source_cache,
+            location.object_store,
+            &location.data_file_path,
+        );
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, position, size, BlobKind::Inline, None),
+        })
+    }
+
+    async fn collect_dedicated(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let blob_id = columns.blob_ids.value(idx);
+        let size = columns.sizes.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
+        let source = shared_blob_source(&mut self.source_cache, location.object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None),
+        })
+    }
+
+    async fn collect_packed(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let blob_id = columns.blob_ids.value(idx);
+        let size = columns.sizes.value(idx);
+        let position = columns.positions.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
+        let source = shared_blob_source(&mut self.source_cache, location.object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, position, size, BlobKind::Packed, None),
+        })
+    }
+
+    async fn collect_external(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let uri_or_path = columns.blob_uris.value(idx).to_string();
+        let position = columns.positions.value(idx);
+        let size = columns.sizes.value(idx);
+        let base_id = columns.blob_ids.value(idx);
+        let (object_store, path) = if base_id == 0 {
+            let registry = self.dataset.session.store_registry();
+            let params = self
+                .dataset
+                .store_params
+                .as_ref()
+                .map(|p| Arc::new((**p).clone()))
+                .unwrap_or_else(|| Arc::new(ObjectStoreParams::default()));
+            ObjectStore::from_uri_and_params(registry, &uri_or_path, &params).await?
+        } else {
+            let object_store = if let Some(store) = self.store_cache.get(&base_id) {
+                store.clone()
+            } else {
+                let store = self.dataset.object_store(Some(base_id)).await?;
+                self.store_cache.insert(base_id, store.clone());
+                store
+            };
+            let base_root = if let Some(path) = self.external_base_path_cache.get(&base_id) {
+                path.clone()
+            } else {
+                let base = self
+                    .dataset
+                    .manifest
+                    .base_paths
+                    .get(&base_id)
+                    .ok_or_else(|| {
+                        Error::invalid_input(format!(
+                            "External blob references unknown base_id {}",
+                            base_id
+                        ))
+                    })?;
+                let path = base.extract_path(self.dataset.session.store_registry())?;
+                self.external_base_path_cache.insert(base_id, path.clone());
+                path
+            };
+            let path = join_base_and_relative_path(&base_root, &uri_or_path)?;
+            (object_store, path)
+        };
+        let size = if size > 0 {
+            size
+        } else {
+            object_store.size(&path).await?
+        };
+        let source = shared_blob_source(&mut self.source_cache, object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(
+                source,
+                position,
+                size,
+                BlobKind::External,
+                Some(uri_or_path),
+            ),
+        })
+    }
+}
+
+fn normalize_external_absolute_uri(uri: &str) -> Result<String> {
+    let url = Url::parse(uri).map_err(|_| {
+        Error::invalid_input(format!(
+            "External URI '{}' is outside registered external bases and is not a valid absolute URI",
+            uri
+        ))
+    })?;
+    Ok(url.to_string())
+}
+
+fn join_base_and_relative_path(base: &Path, relative_path: &str) -> Result<Path> {
+    let relative = Path::parse(relative_path).map_err(|e| {
+        Error::invalid_input(format!(
+            "Invalid relative external blob path '{}': {}",
+            relative_path, e
+        ))
+    })?;
+    Ok(Path::from_iter(base.parts().chain(relative.parts())))
+}
+
+/// Resolve the physical read location for a blob row in a base-aware way.
+///
+/// Given a `row_addr`, this helper locates the owning fragment and the blob field's
+/// data file, then returns the concrete object store and paths needed to read blob
+/// bytes correctly under multi-base datasets.
+///
+/// It uses two caller-provided caches:
+/// - `fragment_cache` memoizes per-fragment path metadata (`data_file_dir`,
+///   `data_file_path`, and `data_file_key`) plus the resolved store.
+/// - `store_cache` memoizes `base_id -> ObjectStore` so multiple fragments that
+///   share the same base do not repeat async store resolution.
+async fn resolve_blob_read_location(
+    dataset: &Arc<Dataset>,
+    blob_field_id: u32,
+    row_addr: u64,
+    fragment_cache: &mut HashMap<u32, BlobReadLocation>,
     store_cache: &mut HashMap<u32, Arc<ObjectStore>>,
 ) -> Result<BlobReadLocation> {
     let frag_id = RowAddress::from(row_addr).fragment_id();
@@ -2257,9 +3596,12 @@ fn data_file_key_from_path(path: &str) -> &str {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
+    use std::collections::{HashMap, VecDeque};
     use std::ops::Range;
-    use std::sync::Arc;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
     use std::time::Duration;
 
     use arrow::{
@@ -2271,11 +3613,12 @@ mod tests {
         Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatchIterator, StringArray,
         StructArray, UInt8Array, UInt32Array, UInt64Array,
     };
+    use arrow_buffer::NullBuffer;
     use arrow_schema::{DataType, Field, Schema};
     use async_trait::async_trait;
     use bytes::Bytes;
     use chrono::Utc;
-    use futures::{StreamExt, TryStreamExt, future::try_join_all};
+    use futures::{StreamExt, TryStreamExt};
     use lance_arrow::{
         ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
         BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
@@ -2295,11 +3638,11 @@ mod tests {
         MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, PutPayload, PutResult,
         path::Path,
     };
-    use tokio::sync::Notify;
+    use tokio::sync::{Notify, Semaphore};
     use url::Url;
 
     use lance_core::{
-        Error, Result,
+        Error, ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION, Result,
         utils::tempfile::{TempDir, TempStrDir},
     };
     use lance_datagen::{BatchCount, RowCount, array};
@@ -2310,15 +3653,17 @@ mod tests {
     use uuid::Uuid;
 
     use super::{
-        BlobEntry, BlobFile, BlobSource, ExternalBaseCandidate, ExternalBaseResolver,
-        ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path,
-        execute_blob_read_plan, plan_blob_read_plans,
+        BlobEntry, BlobFile, BlobRangeRequest, BlobReadRange, BlobSource, ExternalBaseCandidate,
+        ExternalBaseResolver, ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path,
+        execute_blob_entries, execute_blob_read_batches_stream, execute_blob_read_plan,
+        plan_blob_read_batches, plan_blob_read_plans,
     };
     use crate::{
         Dataset,
         blob::{BlobArrayBuilder, BlobDescriptorArrayBuilder, PackedBlobWriter, blob_field},
         dataset::{
             CommitBuilder, ExternalBlobMode, WriteMode, WriteParams,
+            scanner::MaterializationStyle,
             transaction::{DataReplacementGroup, Operation, Transaction},
         },
         utils::test::TestDatasetGenerator,
@@ -2538,8 +3883,12 @@ mod tests {
     #[derive(Debug)]
     struct RecordingRangeObjectStore {
         data: Bytes,
-        gate: Option<Arc<Notify>>,
+        gate: Option<Arc<Semaphore>>,
         requested_ranges: std::sync::Mutex<Vec<Range<u64>>>,
+        started_blob_requests: AtomicUsize,
+        active_blob_requests: AtomicUsize,
+        peak_active_blob_requests: AtomicUsize,
+        request_started: Notify,
     }
 
     impl RecordingRangeObjectStore {
@@ -2548,17 +3897,39 @@ mod tests {
                 data,
                 gate: None,
                 requested_ranges: std::sync::Mutex::new(Vec::new()),
+                started_blob_requests: AtomicUsize::new(0),
+                active_blob_requests: AtomicUsize::new(0),
+                peak_active_blob_requests: AtomicUsize::new(0),
+                request_started: Notify::new(),
             }
         }
 
-        fn with_gate(data: Bytes, gate: Arc<Notify>) -> Self {
+        fn with_gate(data: Bytes, gate: Arc<Semaphore>) -> Self {
             Self {
                 data,
                 gate: Some(gate),
                 requested_ranges: std::sync::Mutex::new(Vec::new()),
+                started_blob_requests: AtomicUsize::new(0),
+                active_blob_requests: AtomicUsize::new(0),
+                peak_active_blob_requests: AtomicUsize::new(0),
+                request_started: Notify::new(),
+            }
+        }
+
+        async fn wait_for_blob_requests(&self, expected: usize) {
+            loop {
+                let request_started = self.request_started.notified();
+                if self.started_blob_requests.load(Ordering::Acquire) >= expected {
+                    return;
+                }
+                request_started.await;
             }
         }
 
+        fn peak_active_blob_requests(&self) -> usize {
+            self.peak_active_blob_requests.load(Ordering::Acquire)
+        }
+
         fn requested_ranges(&self) -> Vec<Range<u64>> {
             self.requested_ranges.lock().unwrap().clone()
         }
@@ -2622,10 +3993,21 @@ mod tests {
                 }
             };
             let is_full_object_probe = range.start == 0 && range.end == self.data.len() as u64;
-            if !is_full_object_probe && let Some(gate) = &self.gate {
-                gate.notified().await;
-            }
             self.requested_ranges.lock().unwrap().push(range.clone());
+            if !is_full_object_probe {
+                let active = self.active_blob_requests.fetch_add(1, Ordering::AcqRel) + 1;
+                self.peak_active_blob_requests
+                    .fetch_max(active, Ordering::AcqRel);
+                self.started_blob_requests.fetch_add(1, Ordering::AcqRel);
+                self.request_started.notify_waiters();
+                if let Some(gate) = &self.gate {
+                    gate.acquire()
+                        .await
+                        .expect("test gate should remain open")
+                        .forget();
+                }
+                self.active_blob_requests.fetch_sub(1, Ordering::AcqRel);
+            }
             let bytes = self.data.slice(range.start as usize..range.end as usize);
             Ok(GetResult {
                 payload: GetResultPayload::Stream(
@@ -2668,9 +4050,10 @@ mod tests {
         }
     }
 
-    fn recording_range_store_with_url(
+    fn recording_range_store_with_url_and_block_size(
         data: Bytes,
         url: &str,
+        block_size: Option<usize>,
     ) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
         const TEST_RANGE_STORE_SIZE: usize = 128 * 1024;
         let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())];
@@ -2679,7 +4062,7 @@ mod tests {
         let store = Arc::new(ObjectStore::new(
             inner.clone() as Arc<dyn object_store::ObjectStore>,
             Url::parse(url).unwrap(),
-            None,
+            block_size,
             None,
             false,
             true,
@@ -2690,22 +4073,40 @@ mod tests {
         (store, inner)
     }
 
+    fn recording_range_store_with_url(
+        data: Bytes,
+        url: &str,
+    ) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
+        recording_range_store_with_url_and_block_size(data, url, None)
+    }
+
     fn recording_range_store(data: Bytes) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
         recording_range_store_with_url(data, "mock://recording/blob-range-tests")
     }
 
+    fn recording_range_store_with_block_size(
+        data: Bytes,
+        block_size: usize,
+    ) -> (Arc<ObjectStore>, Arc<RecordingRangeObjectStore>) {
+        recording_range_store_with_url_and_block_size(
+            data,
+            "mock://recording/blob-range-tests",
+            Some(block_size),
+        )
+    }
+
     fn gated_range_store(
         data: Bytes,
         url: &str,
     ) -> (
         Arc<ObjectStore>,
         Arc<RecordingRangeObjectStore>,
-        Arc<Notify>,
+        Arc<Semaphore>,
     ) {
         const TEST_RANGE_STORE_SIZE: usize = 128 * 1024;
         let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())];
         padded[..data.len()].copy_from_slice(data.as_ref());
-        let gate = Arc::new(Notify::new());
+        let gate = Arc::new(Semaphore::new(0));
         let inner = Arc::new(RecordingRangeObjectStore::with_gate(
             Bytes::from(padded),
             gate.clone(),
@@ -2843,7 +4244,7 @@ mod tests {
         for (actual_idx, (expected_batch_idx, expected_row_idx)) in
             [(5, 5), (6, 7), (8, 3)].iter().enumerate()
         {
-            let val = blobs[actual_idx].read().await.unwrap();
+            let val = blobs[actual_idx].as_ref().unwrap().read().await.unwrap();
             let expected = fixture.data[*expected_batch_idx]
                 .column(1)
                 .as_binary::<i64>()
@@ -2889,7 +4290,7 @@ mod tests {
                 .column(1)
                 .as_binary::<i64>()
                 .value(*expected_row_idx);
-            assert_eq!(blobs[actual_idx].data.as_ref(), expected);
+            assert_eq!(blobs[actual_idx].data.as_deref(), Some(expected));
         }
     }
 
@@ -2924,6 +4325,8 @@ mod tests {
         let blobs2 = fixture.dataset.take_blobs(&row_ids, "blobs").await.unwrap();
 
         for (blob1, blob2) in blobs.iter().zip(blobs2.iter()) {
+            let blob1 = blob1.as_ref().unwrap();
+            let blob2 = blob2.as_ref().unwrap();
             assert_eq!(blob1.position(), blob2.position());
             assert_eq!(blob1.size(), blob2.size());
             assert_eq!(blob1.data_path(), blob2.data_path());
@@ -3079,7 +4482,7 @@ mod tests {
 
         // Verify we can read the blob content
         for blob in &blobs {
-            let content = blob.read().await.unwrap();
+            let content = blob.as_ref().unwrap().read().await.unwrap();
             assert!(!content.is_empty(), "Blob content should not be empty");
         }
 
@@ -3092,7 +4495,7 @@ mod tests {
 
         // Verify we can read the blob content from second fragment
         for blob in &blobs {
-            let content = blob.read().await.unwrap();
+            let content = blob.as_ref().unwrap().read().await.unwrap();
             assert!(!content.is_empty(), "Blob content should not be empty");
         }
 
@@ -3280,7 +4683,10 @@ mod tests {
             .unwrap();
         assert_eq!(blob_files.len(), payloads.len());
         for (blob_file, expected) in blob_files.iter().zip(payloads) {
-            assert_eq!(blob_file.read().await.unwrap().as_ref(), expected);
+            assert_eq!(
+                blob_file.as_ref().unwrap().read().await.unwrap().as_ref(),
+                expected
+            );
         }
 
         let read_blobs = dataset
@@ -3292,7 +4698,7 @@ mod tests {
             .unwrap();
         assert_eq!(read_blobs.len(), payloads.len());
         for (read_blob, expected) in read_blobs.iter().zip(payloads) {
-            assert_eq!(read_blob.data.as_ref(), expected);
+            assert_eq!(read_blob.data.as_deref(), Some(expected));
         }
     }
 
@@ -3338,8 +4744,8 @@ mod tests {
             .unwrap();
 
         assert_eq!(blobs.len(), 2);
-        let first = blobs[0].read().await.unwrap();
-        let second = blobs[1].read().await.unwrap();
+        let first = blobs[0].as_ref().unwrap().read().await.unwrap();
+        let second = blobs[1].as_ref().unwrap().read().await.unwrap();
         assert_eq!(first.as_ref(), b"hello");
         assert_eq!(second.as_ref(), b"world");
     }
@@ -3389,7 +4795,10 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-inline");
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"prepared-inline"
+        );
     }
 
     #[tokio::test]
@@ -3489,8 +4898,9 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-packed");
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.read().await.unwrap().as_ref(), b"prepared-packed");
+        assert_eq!(blob.kind(), BlobKind::Packed);
     }
 
     #[tokio::test]
@@ -3571,8 +4981,14 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(blobs.len(), 2);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"initial");
-        assert_eq!(blobs[1].read().await.unwrap().as_ref(), b"append");
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"initial"
+        );
+        assert_eq!(
+            blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"append"
+        );
     }
 
     #[tokio::test]
@@ -3672,10 +5088,10 @@ mod tests {
             .unwrap();
         assert_eq!(blobs.len(), 1);
         assert_eq!(
-            blobs[0].read().await.unwrap().as_ref(),
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
             b"nested-replacement"
         );
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        assert_eq!(blobs[0].as_ref().unwrap().kind(), BlobKind::Packed);
     }
 
     #[tokio::test]
@@ -3750,35 +5166,826 @@ mod tests {
             },
         )
         .unwrap();
-        file_writer.write_batch(&replacement_batch).await.unwrap();
-        file_writer.finish().await.unwrap();
-
-        let data_file = dataset
-            .create_data_file(&data_file_name, None)
-            .await
-            .unwrap();
-
-        assert_eq!(data_file.fields.len(), 2);
-        assert_eq!(data_file.column_indices.as_ref(), &[0, 1]);
+        file_writer.write_batch(&replacement_batch).await.unwrap();
+        file_writer.finish().await.unwrap();
+
+        let data_file = dataset
+            .create_data_file(&data_file_name, None)
+            .await
+            .unwrap();
+
+        assert_eq!(data_file.fields.len(), 2);
+        assert_eq!(data_file.column_indices.as_ref(), &[0, 1]);
+    }
+
+    #[tokio::test]
+    async fn test_write_and_take_nested_blob_v2() {
+        let test_dir = TempStrDir::default();
+        let packed_payload = vec![0x4A; super::INLINE_MAX + 1024];
+
+        let mut blob_builder = BlobArrayBuilder::new(3);
+        blob_builder.push_bytes(b"hello").unwrap();
+        blob_builder.push_bytes(&packed_payload).unwrap();
+        blob_builder.push_null().unwrap();
+        let blob_array: ArrayRef = blob_builder.finish().unwrap();
+
+        let (schema, batch) = nested_blob_v2_batch(blob_array);
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let info_batch = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let blob_desc = info_batch
+            .column(0)
+            .as_struct()
+            .column_by_name("blob")
+            .unwrap()
+            .as_struct();
+        assert_eq!(
+            blob_desc
+                .column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Inline as u8
+        );
+        assert_eq!(
+            blob_desc
+                .column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(1),
+            BlobKind::Packed as u8
+        );
+
+        let blobs = dataset
+            .take_blobs_by_indices(&[0, 1], "info.blob")
+            .await
+            .unwrap();
+        assert_eq!(blobs.len(), 2);
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"hello"
+        );
+        assert_eq!(
+            blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(),
+            packed_payload.as_slice()
+        );
+
+        let null_blobs = dataset
+            .take_blobs_by_indices(&[2], "info.blob")
+            .await
+            .unwrap();
+        assert_eq!(null_blobs.len(), 1);
+        assert!(null_blobs[0].is_none());
+
+        let filtered = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .filter("info.blob IS NOT NULL")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(filtered.num_rows(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_write_and_scan_list_blob_v2_descriptions() {
+        let test_dir = TempStrDir::default();
+        let packed_payload = vec![0x4B; super::INLINE_MAX + 1024];
+
+        let mut blob_builder = BlobArrayBuilder::new(4);
+        blob_builder.push_bytes(b"hello").unwrap();
+        blob_builder.push_null().unwrap();
+        blob_builder.push_bytes(&packed_payload).unwrap();
+        blob_builder.push_bytes(b"tail").unwrap();
+        let blob_values = blob_builder.finish().unwrap();
+
+        let item_field = Arc::new(blob_field("item", true));
+        let list_array: ArrayRef = Arc::new(
+            arrow_array::ListArray::try_new(
+                item_field.clone(),
+                arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![
+                    0i32, 3, 3, 3, 4,
+                ])),
+                blob_values,
+                Some(arrow_buffer::NullBuffer::from(vec![
+                    true, true, false, true,
+                ])),
+            )
+            .unwrap(),
+        );
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("blobs", DataType::List(item_field), true),
+            Field::new("id", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![list_array, Arc::new(Int32Array::from(vec![0, 1, 2, 3]))],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    enable_stable_row_ids: true,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let descriptions = dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let lists = descriptions.column(0).as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]);
+        assert!(lists.is_valid(0));
+        assert!(lists.is_valid(1));
+        assert!(lists.is_null(2));
+        assert!(lists.is_valid(3));
+
+        let DataType::List(descriptor_field) = lists.data_type() else {
+            panic!("unexpected list type: {}", lists.data_type());
+        };
+        assert!(matches!(descriptor_field.data_type(), DataType::Struct(_)));
+        assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let descriptors = lists.values().as_struct();
+        assert_eq!(descriptors.fields().len(), 5);
+        assert_eq!(descriptors.fields()[0].name(), "kind");
+        assert!(descriptors.is_valid(0));
+        assert!(descriptors.is_null(1));
+        assert!(descriptors.is_valid(2));
+        assert!(descriptors.is_valid(3));
+        let kinds = descriptors
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<UInt8Type>();
+        assert_eq!(kinds.value(0), BlobKind::Inline as u8);
+        assert_eq!(kinds.value(2), BlobKind::Packed as u8);
+        assert_eq!(kinds.value(3), BlobKind::Inline as u8);
+
+        let filtered = dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .filter("blobs IS NOT NULL")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(filtered.num_rows(), 3);
+
+        let mut scanner = dataset.scan();
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let bytes = scanner
+            .project(&["blobs"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let lists = bytes.column(0).as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]);
+        assert!(lists.is_valid(0));
+        assert!(lists.is_valid(1));
+        assert!(lists.is_null(2));
+        assert!(lists.is_valid(3));
+        let DataType::List(value_field) = lists.data_type() else {
+            panic!("unexpected list type: {}", lists.data_type());
+        };
+        assert_eq!(value_field.data_type(), &DataType::LargeBinary);
+        assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let values = lists.values().as_binary::<i64>();
+        assert_eq!(values.value(0), b"hello");
+        assert!(values.is_null(1));
+        assert_eq!(values.value(2), packed_payload.as_slice());
+        assert_eq!(values.value(3), b"tail");
+
+        for (filter, materialization_style) in [
+            (None, MaterializationStyle::Heuristic),
+            (Some("id >= 2"), MaterializationStyle::Heuristic),
+            (Some("id >= 2"), MaterializationStyle::AllEarly),
+        ] {
+            let mut scanner = dataset.scan();
+            scanner.blob_handling(BlobHandling::AllBinary);
+            scanner.materialization_style(materialization_style);
+            scanner
+                .project(&["blobs", ROW_LAST_UPDATED_AT_VERSION, ROW_CREATED_AT_VERSION])
+                .unwrap()
+                .with_row_id()
+                .with_row_address();
+            if let Some(filter) = filter {
+                scanner.filter(filter).unwrap();
+            }
+
+            let expected_schema = scanner.schema().await.unwrap();
+            let batch = scanner.try_into_batch().await.unwrap();
+            assert_eq!(batch.schema().as_ref(), expected_schema.as_ref());
+            assert_eq!(batch.num_rows(), if filter.is_some() { 2 } else { 4 });
+            for column in [
+                ROW_ID,
+                ROW_ADDR,
+                ROW_LAST_UPDATED_AT_VERSION,
+                ROW_CREATED_AT_VERSION,
+            ] {
+                assert!(
+                    batch.column_by_name(column).is_some(),
+                    "requested system column {column} was missing"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_and_scan_struct_nested_list_blob_v2() {
+        let test_dir = TempStrDir::default();
+
+        let mut blob_builder = BlobArrayBuilder::new(2);
+        blob_builder.push_bytes(b"nested").unwrap();
+        blob_builder.push_null().unwrap();
+        let blob_values = blob_builder.finish().unwrap();
+
+        let item_field = Arc::new(blob_field("item", true));
+        let list_field = Field::new("blobs", DataType::List(item_field.clone()), true);
+        let list_array: ArrayRef = Arc::new(
+            arrow_array::ListArray::try_new(
+                item_field,
+                arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![0i32, 2, 2])),
+                blob_values,
+                None,
+            )
+            .unwrap(),
+        );
+        let info_fields = vec![Field::new("name", DataType::Utf8, false), list_field];
+        let info_array: ArrayRef = Arc::new(
+            StructArray::try_new(
+                info_fields.clone().into(),
+                vec![
+                    Arc::new(StringArray::from(vec!["row-0", "row-1"])) as ArrayRef,
+                    list_array,
+                ],
+                None,
+            )
+            .unwrap(),
+        );
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "info",
+            DataType::Struct(info_fields.into()),
+            true,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let descriptions = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let info = descriptions.column(0).as_struct();
+        assert_eq!(
+            info.column_by_name("name")
+                .unwrap()
+                .as_string::<i32>()
+                .value(0),
+            "row-0"
+        );
+        let lists = info.column_by_name("blobs").unwrap().as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]);
+        let DataType::List(descriptor_field) = lists.data_type() else {
+            panic!("unexpected nested list type: {}", lists.data_type());
+        };
+        assert!(matches!(descriptor_field.data_type(), DataType::Struct(_)));
+        assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let descriptors = lists.values().as_struct();
+        assert_eq!(descriptors.fields().len(), 5);
+        assert!(descriptors.is_valid(0));
+        assert!(descriptors.is_null(1));
+
+        let mut scanner = dataset.scan();
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let bytes = scanner
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let info = bytes.column(0).as_struct();
+        let lists = info.column_by_name("blobs").unwrap().as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]);
+        let DataType::List(value_field) = lists.data_type() else {
+            panic!("unexpected nested list type: {}", lists.data_type());
+        };
+        assert_eq!(value_field.data_type(), &DataType::LargeBinary);
+        assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let values = lists.values().as_binary::<i64>();
+        assert_eq!(values.value(0), b"nested");
+        assert!(values.is_null(1));
+    }
+
+    #[tokio::test]
+    async fn test_nested_blob_v2_requires_v2_2() {
+        let test_dir = TempStrDir::default();
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_bytes(b"hello").unwrap();
+        let blob_array: ArrayRef = blob_builder.finish().unwrap();
+
+        let (schema, batch) = nested_blob_v2_batch(blob_array);
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let result = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await;
+
+        assert!(
+            result.is_err(),
+            "Nested blob v2 should be rejected for file version 2.1"
+        );
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Blob v2 requires file version >= 2.2")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_empty_range_returns_empty_bytes() {
+        let store = reject_empty_range_store();
+        let path = Path::from("blobs/test.bin");
+
+        let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0);
+        assert!(empty_blob.read().await.unwrap().is_empty());
+        assert!(empty_blob.read_up_to(16).await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_tracks_relative_cursor() {
+        let test_dir = TempDir::default();
+        let file_path = test_dir.std_path().join("blob.bin");
+        std::fs::write(&file_path, b"abcd").unwrap();
+
+        let path = Path::from_absolute_path(file_path).unwrap();
+        let blob = BlobFile::new_packed(Arc::new(ObjectStore::local()), path, 1, 2);
+
+        assert_eq!(blob.read().await.unwrap().as_ref(), b"bc");
+        assert_eq!(blob.tell().await.unwrap(), 2);
+        assert!(blob.read().await.unwrap().is_empty());
+        assert!(blob.read_up_to(1).await.unwrap().is_empty());
+        assert_eq!(blob.tell().await.unwrap(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_range_does_not_change_cursor() {
+        let (store, _) = recording_range_store(Bytes::from_static(b"abcdefgh"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 1, 6);
+
+        let bytes = blob.read_range(2..5).await.unwrap();
+        assert_eq!(bytes.as_ref(), b"def");
+        assert_eq!(blob.tell().await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_ranges_preserves_input_order() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 1, 6);
+
+        let chunks = blob.read_ranges(&[4..6, 0..2, 2..4, 2..2]).await.unwrap();
+        assert_eq!(chunks[0].as_ref(), b"fg");
+        assert_eq!(chunks[1].as_ref(), b"bc");
+        assert_eq!(chunks[2].as_ref(), b"de");
+        assert!(chunks[3].is_empty());
+        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_read_range_rejects_out_of_bounds() {
+        let (store, _) = recording_range_store(Bytes::from_static(b"abcdef"));
+        let path = Path::from("blobs/test.bin");
+        let blob = BlobFile::new_packed(store, path, 0, 4);
+
+        let err = blob.read_range(1..5).await.unwrap_err();
+        assert!(err.to_string().contains("exceeds blob size"));
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_staggered_same_source_ranges_run_concurrently() {
+        let (store, inner, gate) = gated_range_store(
+            Bytes::from_static(b"abcdefgh"),
+            "mock://same-source/blob-range-tests",
+        );
+        let blob = Arc::new(BlobFile::new_dedicated(
+            store,
+            Path::from("blobs/test.bin"),
+            8,
+        ));
+
+        let first_blob = blob.clone();
+        let first = tokio::spawn(async move { first_blob.read_range(0..3).await });
+        inner.wait_for_blob_requests(1).await;
+
+        let second_blob = blob.clone();
+        let second = tokio::spawn(async move { second_blob.read_range(4..7).await });
+        tokio::time::timeout(Duration::from_secs(1), inner.wait_for_blob_requests(2))
+            .await
+            .expect("the second same-source range should start while the first is in flight");
+
+        assert_eq!(inner.peak_active_blob_requests(), 2);
+        gate.add_permits(2);
+        assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc");
+        assert_eq!(second.await.unwrap().unwrap().as_ref(), b"efg");
+    }
+
+    #[tokio::test]
+    async fn test_blob_file_staggered_multiple_source_ranges_run_concurrently() {
+        let (first_store, first_inner, first_gate) = gated_range_store(
+            Bytes::from_static(b"abcdefgh"),
+            "mock://first-source/blob-range-tests",
+        );
+        let (second_store, second_inner, second_gate) = gated_range_store(
+            Bytes::from_static(b"ijklmnop"),
+            "mock://second-source/blob-range-tests",
+        );
+        let first_blob = Arc::new(BlobFile::new_dedicated(
+            first_store,
+            Path::from("blobs/first.bin"),
+            8,
+        ));
+        let second_blob = Arc::new(BlobFile::new_dedicated(
+            second_store,
+            Path::from("blobs/second.bin"),
+            8,
+        ));
+
+        let first = tokio::spawn(async move { first_blob.read_range(0..3).await });
+        first_inner.wait_for_blob_requests(1).await;
+
+        let second = tokio::spawn(async move { second_blob.read_range(4..7).await });
+        tokio::time::timeout(
+            Duration::from_secs(1),
+            second_inner.wait_for_blob_requests(1),
+        )
+        .await
+        .expect("a read from another source should start while the first is in flight");
+
+        first_gate.add_permits(1);
+        second_gate.add_permits(1);
+        assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc");
+        assert_eq!(second.await.unwrap().unwrap().as_ref(), b"mno");
+    }
+
+    #[tokio::test]
+    async fn test_blob_files_share_source_and_coalesce() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
+        let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None);
+        let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None);
+
+        let (data1, data2) = tokio::join!(blob1.read(), blob2.read());
+        assert_eq!(data1.unwrap().as_ref(), b"bcd");
+        assert_eq!(data2.unwrap().as_ref(), b"efg");
+        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
+    }
+
+    #[tokio::test]
+    async fn test_execute_blob_entries_preserves_order_and_coalesces() {
+        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
+        let entries = vec![
+            BlobEntry {
+                selection_index: 0,
+                row_address: 10,
+                file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None),
+            },
+            BlobEntry {
+                selection_index: 1,
+                row_address: 11,
+                file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None),
+            },
+        ];
+        let mut blobs = execute_blob_entries(entries, 2, None).await.unwrap();
+        blobs.sort_by_key(|blob| blob.selection_index);
+
+        assert_eq!(blobs.len(), 2);
+        assert_eq!(blobs[0].row_address, 10);
+        assert_eq!(blobs[0].data.as_ref(), b"efg");
+        assert_eq!(blobs[1].row_address, 11);
+        assert_eq!(blobs[1].data.as_ref(), b"bcd");
+        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
+    }
+
+    #[test]
+    fn test_blob_read_batches_bound_physical_bytes() {
+        let (store, _) = recording_range_store(Bytes::from_static(b"abcdefghij"));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
+        let entries = vec![
+            BlobEntry {
+                selection_index: 1,
+                row_address: 11,
+                file: BlobFile::with_source(source.clone(), 4, 6, BlobKind::Packed, None),
+            },
+            BlobEntry {
+                selection_index: 0,
+                row_address: 10,
+                file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None),
+            },
+        ];
+        let requested = [BlobReadRange::new(0, 3), BlobReadRange::new(0, 5)];
+
+        let batches = plan_blob_read_batches(entries, Some(&requested), 4).unwrap();
+
+        assert_eq!(batches.len(), 2);
+        assert_eq!(batches[0].expected_selection_indices, VecDeque::from([0]));
+        assert_eq!(batches[1].expected_selection_indices, VecDeque::from([1]));
+    }
+
+    #[tokio::test]
+    async fn test_blob_read_batches_bound_coalesced_physical_bytes() {
+        const BLOCK_SIZE: usize = 64 * 1024;
+        const RANGE_GAP: u64 = 32 * 1024;
+        const REQUEST_COUNT: usize = 4;
+
+        let mut data = vec![0_u8; 128 * 1024];
+        for request_index in 0..REQUEST_COUNT {
+            data[request_index * RANGE_GAP as usize] = request_index as u8 + 1;
+        }
+        let (store, inner) = recording_range_store_with_block_size(Bytes::from(data), BLOCK_SIZE);
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/dense.pack")));
+        let entries = (0..REQUEST_COUNT)
+            .map(|request_index| BlobEntry {
+                selection_index: request_index,
+                row_address: request_index as u64,
+                file: BlobFile::with_source(
+                    source.clone(),
+                    request_index as u64 * RANGE_GAP,
+                    1,
+                    BlobKind::Packed,
+                    None,
+                ),
+            })
+            .collect();
+        let requested = [BlobReadRange::new(0, 1); REQUEST_COUNT];
+
+        let batches =
+            plan_blob_read_batches(entries, Some(&requested), REQUEST_COUNT as u64).unwrap();
+        assert_eq!(batches.len(), REQUEST_COUNT);
+
+        let results = execute_blob_read_batches_stream(
+            batches,
+            Arc::new(ReadBlobsExecution::new(Some(REQUEST_COUNT as u64))),
+            REQUEST_COUNT,
+            true,
+        )
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+
+        assert_eq!(results.len(), REQUEST_COUNT);
+        for (request_index, result) in results.iter().enumerate() {
+            assert_eq!(result.selection_index, request_index);
+            assert_eq!(result.data.as_ref(), &[request_index as u8 + 1]);
+        }
+        assert_eq!(
+            inner.requested_blob_ranges(),
+            (0..REQUEST_COUNT)
+                .map(|request_index| {
+                    let start = request_index as u64 * RANGE_GAP;
+                    start..start + 1
+                })
+                .collect::<Vec<_>>()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_blob_ranges_preserves_request_order_across_fragments() {
+        let fixture = BlobTestFixture::new().await;
+        let selections = [
+            (22_u64, 2_usize, 2_usize),
+            (2, 0, 2),
+            (12, 1, 2),
+            (12, 1, 2),
+        ];
+        let ranges = vec![
+            BlobReadRange::new(2, 3),
+            BlobReadRange::new(0, 2),
+            BlobReadRange::new(3, 2),
+            BlobReadRange::new(0, 3),
+        ];
+
+        let results = fixture
+            .dataset
+            .read_blob_ranges("blobs")
+            .unwrap()
+            .with_row_indices(
+                selections
+                    .iter()
+                    .zip(&ranges)
+                    .map(|((row_index, _, _), range)| {
+                        BlobRangeRequest::new(*row_index, range.offset, range.length)
+                    })
+                    .collect::<Vec<_>>(),
+            )
+            .execute()
+            .await
+            .unwrap();
+
+        assert_eq!(results.len(), selections.len());
+        for (request_index, (((_, batch_index, row_index), range), result)) in
+            selections.iter().zip(&ranges).zip(&results).enumerate()
+        {
+            let payload = fixture.data[*batch_index]
+                .column(1)
+                .as_binary::<i64>()
+                .value(*row_index);
+            let start = range.offset as usize;
+            let end = start + range.length as usize;
+            assert_eq!(result.request_index, request_index);
+            assert_eq!(result.range, *range);
+            assert_eq!(result.data.as_deref(), Some(&payload[start..end]));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_blob_selection_apis_preserve_v1_nulls_and_empty_values() {
+        let test_dir = TempStrDir::default();
+        let blob_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("blob", DataType::LargeBinary, true).with_metadata(blob_metadata),
+        ]));
+        let blobs =
+            LargeBinaryArray::from(vec![None, Some(b"".as_slice()), Some(b"abc".as_slice())]);
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(blobs)]).unwrap();
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_0),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let blob_files = dataset
+            .take_blobs_by_indices(&[2, 1, 0], "blob")
+            .await
+            .unwrap();
+        assert_eq!(blob_files.len(), 3);
+        assert_eq!(
+            blob_files[0]
+                .as_ref()
+                .unwrap()
+                .read()
+                .await
+                .unwrap()
+                .as_ref(),
+            b"abc"
+        );
+        assert!(
+            blob_files[1]
+                .as_ref()
+                .unwrap()
+                .read()
+                .await
+                .unwrap()
+                .is_empty()
+        );
+        assert!(blob_files[2].is_none());
+
+        let blobs = dataset
+            .read_blobs("blob")
+            .unwrap()
+            .with_row_indices([2, 1, 0])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(
+            blobs
+                .iter()
+                .map(|blob| (blob.row_address, blob.data.as_deref()))
+                .collect::<Vec<_>>(),
+            vec![
+                (2, Some(b"abc".as_slice())),
+                (1, Some(b"".as_slice())),
+                (0, None),
+            ]
+        );
+
+        let results = dataset
+            .read_blob_ranges("blob")
+            .unwrap()
+            .with_row_indices([
+                BlobRangeRequest::new(0, 0, 0),
+                BlobRangeRequest::new(0, 0, 1),
+                BlobRangeRequest::new(1, 0, 0),
+                BlobRangeRequest::new(2, 1, 1),
+            ])
+            .execute()
+            .await
+            .unwrap();
+
+        assert_eq!(
+            results
+                .iter()
+                .map(|result| (result.request_index, result.data.as_deref()))
+                .collect::<Vec<_>>(),
+            vec![
+                (0, None),
+                (1, None),
+                (2, Some(b"".as_slice())),
+                (3, Some(b"b".as_slice())),
+            ]
+        );
+
+        let descriptions = StructArray::try_new(
+            vec![
+                Arc::new(Field::new("position", DataType::UInt64, false)),
+                Arc::new(Field::new("size", DataType::UInt64, false)),
+            ]
+            .into(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0])) as ArrayRef,
+                Arc::new(UInt64Array::from(vec![0])) as ArrayRef,
+            ],
+            Some(NullBuffer::from(vec![false])),
+        )
+        .unwrap();
+        let row_addrs = UInt64Array::from(vec![u64::MAX]);
+        assert!(
+            collect_blob_entries_v1(&dataset, u32::MAX, &descriptions, &row_addrs)
+                .unwrap()
+                .is_empty()
+        );
     }
 
     #[tokio::test]
-    async fn test_write_and_take_nested_blob_v2() {
+    async fn test_blob_selection_apis_preserve_v2_nulls_and_empty_values() {
         let test_dir = TempStrDir::default();
-        let packed_payload = vec![0x4A; super::INLINE_MAX + 1024];
-
         let mut blob_builder = BlobArrayBuilder::new(3);
-        blob_builder.push_bytes(b"hello").unwrap();
-        blob_builder.push_bytes(&packed_payload).unwrap();
+        blob_builder.push_empty().unwrap();
         blob_builder.push_null().unwrap();
-        let blob_array: ArrayRef = blob_builder.finish().unwrap();
-
-        let (schema, batch) = nested_blob_v2_batch(blob_array);
-        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
-
+        blob_builder.push_bytes(b"abc").unwrap();
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![blob_builder.finish().unwrap()]).unwrap();
         let dataset = Arc::new(
             Dataset::write(
-                reader,
+                RecordBatchIterator::new(vec![Ok(batch)], schema),
                 &test_dir,
                 Some(WriteParams {
                     data_storage_version: Some(LanceFileVersion::V2_2),
@@ -3789,194 +5996,271 @@ mod tests {
             .unwrap(),
         );
 
-        let info_batch = dataset
-            .scan()
-            .project(&["info"])
-            .unwrap()
-            .try_into_batch()
+        let blob_files = dataset
+            .take_blobs_by_indices(&[2, 1, 0], "blob")
             .await
             .unwrap();
-        let blob_desc = info_batch
-            .column(0)
-            .as_struct()
-            .column_by_name("blob")
-            .unwrap()
-            .as_struct();
+        assert_eq!(blob_files.len(), 3);
         assert_eq!(
-            blob_desc
-                .column_by_name("kind")
+            blob_files[0]
+                .as_ref()
                 .unwrap()
-                .as_primitive::<UInt8Type>()
-                .value(0),
-            BlobKind::Inline as u8
+                .read()
+                .await
+                .unwrap()
+                .as_ref(),
+            b"abc"
         );
-        assert_eq!(
-            blob_desc
-                .column_by_name("kind")
+        assert!(blob_files[1].is_none());
+        assert!(
+            blob_files[2]
+                .as_ref()
                 .unwrap()
-                .as_primitive::<UInt8Type>()
-                .value(1),
-            BlobKind::Packed as u8
+                .read()
+                .await
+                .unwrap()
+                .is_empty()
         );
 
         let blobs = dataset
-            .take_blobs_by_indices(&[0, 1], "info.blob")
+            .read_blobs("blob")
+            .unwrap()
+            .with_row_indices([2, 1, 0])
+            .execute()
             .await
             .unwrap();
-        assert_eq!(blobs.len(), 2);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello");
         assert_eq!(
-            blobs[1].read().await.unwrap().as_ref(),
-            packed_payload.as_slice()
+            blobs
+                .iter()
+                .map(|blob| (blob.row_address, blob.data.as_deref()))
+                .collect::<Vec<_>>(),
+            vec![
+                (2, Some(b"abc".as_slice())),
+                (1, None),
+                (0, Some(b"".as_slice())),
+            ]
         );
 
-        let null_blobs = dataset
-            .take_blobs_by_indices(&[2], "info.blob")
+        let results = dataset
+            .read_blob_ranges("blob")
+            .unwrap()
+            .with_row_indices([
+                BlobRangeRequest::new(0, 0, 0),
+                BlobRangeRequest::new(1, 0, 0),
+                BlobRangeRequest::new(1, 0, 1),
+                BlobRangeRequest::new(2, 1, 1),
+            ])
+            .execute()
             .await
             .unwrap();
-        assert!(null_blobs.is_empty());
+
+        assert_eq!(results.len(), 4);
+        assert_eq!(results[0].request_index, 0);
+        assert_eq!(results[0].data.as_deref(), Some(b"".as_slice()));
+        assert_eq!(results[1].request_index, 1);
+        assert!(results[1].data.is_none());
+        assert_eq!(results[2].request_index, 2);
+        assert!(results[2].data.is_none());
+        assert_eq!(results[3].request_index, 3);
+        assert_eq!(results[3].data.as_deref(), Some(b"b".as_slice()));
     }
 
     #[tokio::test]
-    async fn test_nested_blob_v2_requires_v2_2() {
-        let test_dir = TempStrDir::default();
-
-        let mut blob_builder = BlobArrayBuilder::new(1);
-        blob_builder.push_bytes(b"hello").unwrap();
-        let blob_array: ArrayRef = blob_builder.finish().unwrap();
-
-        let (schema, batch) = nested_blob_v2_batch(blob_array);
-        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    async fn test_read_blob_ranges_handles_overlaps_across_scheduler_splits() {
+        let max_iop_size = *lance_io::object_store::DEFAULT_MAX_IOP_SIZE;
+        let value_size = max_iop_size + 12;
+        let nested_start = value_size / 2 - 1;
+        let nested_end = value_size;
+        let (store, inner) = recording_range_store(Bytes::from(vec![7; value_size as usize]));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/overlapping.bin")));
+        let entries = vec![
+            BlobEntry {
+                selection_index: 0,
+                row_address: 10,
+                file: BlobFile::with_source(
+                    source.clone(),
+                    0,
+                    value_size,
+                    BlobKind::Dedicated,
+                    None,
+                ),
+            },
+            BlobEntry {
+                selection_index: 1,
+                row_address: 10,
+                file: BlobFile::with_source(source, 0, value_size, BlobKind::Dedicated, None),
+            },
+        ];
+        let requested = [
+            BlobReadRange::new(0, value_size),
+            BlobReadRange::new(nested_start, nested_end - nested_start),
+        ];
 
-        let result = Dataset::write(
-            reader,
-            &test_dir,
-            Some(WriteParams {
-                data_storage_version: Some(LanceFileVersion::V2_1),
-                ..Default::default()
-            }),
+        let mut plans = plan_blob_read_plans(entries, Some(&requested)).unwrap();
+        let results = execute_blob_read_plan(
+            plans.pop().unwrap(),
+            Arc::new(ReadBlobsExecution::new(None)),
         )
-        .await;
+        .await
+        .unwrap();
 
-        assert!(
-            result.is_err(),
-            "Nested blob v2 should be rejected for file version 2.1"
+        assert_eq!(results.len(), 2);
+        assert_eq!(results[0].selection_index, 0);
+        assert_eq!(results[0].data.len(), value_size as usize);
+        assert!(results[0].data.iter().all(|byte| *byte == 7));
+        assert_eq!(results[1].selection_index, 1);
+        assert_eq!(results[1].data.len(), (nested_end - nested_start) as usize);
+        assert!(results[1].data.iter().all(|byte| *byte == 7));
+
+        let mut physical_ranges = inner.requested_blob_ranges();
+        physical_ranges.sort_by_key(|range| range.start);
+        assert_eq!(
+            physical_ranges.len(),
+            value_size.div_ceil(max_iop_size) as usize
         );
+        assert_eq!(physical_ranges[0].start, 0);
+        assert_eq!(physical_ranges.last().unwrap().end, value_size);
         assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Blob v2 requires file version >= 2.2")
+            physical_ranges
+                .windows(2)
+                .all(|ranges| ranges[0].end == ranges[1].start)
         );
     }
 
     #[tokio::test]
-    async fn test_blob_file_read_empty_range_returns_empty_bytes() {
-        let store = reject_empty_range_store();
-        let path = Path::from("blobs/test.bin");
-
-        let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0);
-        assert!(empty_blob.read().await.unwrap().is_empty());
-        assert!(empty_blob.read_up_to(16).await.unwrap().is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_blob_file_read_tracks_relative_cursor() {
-        let test_dir = TempDir::default();
-        let file_path = test_dir.std_path().join("blob.bin");
-        std::fs::write(&file_path, b"abcd").unwrap();
-
-        let path = Path::from_absolute_path(file_path).unwrap();
-        let blob = BlobFile::new_packed(Arc::new(ObjectStore::local()), path, 1, 2);
-
-        assert_eq!(blob.read().await.unwrap().as_ref(), b"bc");
-        assert_eq!(blob.tell().await.unwrap(), 2);
-        assert!(blob.read().await.unwrap().is_empty());
-        assert!(blob.read_up_to(1).await.unwrap().is_empty());
-        assert_eq!(blob.tell().await.unwrap(), 2);
-    }
-
-    #[tokio::test]
-    async fn test_blob_file_read_range_does_not_change_cursor() {
-        let (store, _) = recording_range_store(Bytes::from_static(b"abcdefgh"));
-        let path = Path::from("blobs/test.bin");
-        let blob = BlobFile::new_packed(store, path, 1, 6);
-
-        let bytes = blob.read_range(2..5).await.unwrap();
-        assert_eq!(bytes.as_ref(), b"def");
-        assert_eq!(blob.tell().await.unwrap(), 0);
-    }
-
-    #[tokio::test]
-    async fn test_blob_file_read_ranges_preserves_input_order() {
-        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
-        let path = Path::from("blobs/test.bin");
-        let blob = BlobFile::new_packed(store, path, 1, 6);
+    async fn test_read_blob_ranges_rejects_overflow_and_out_of_bounds() {
+        let fixture = BlobTestFixture::new().await;
 
-        let chunks = blob.read_ranges(&[4..6, 0..2, 2..4, 2..2]).await.unwrap();
-        assert_eq!(chunks[0].as_ref(), b"fg");
-        assert_eq!(chunks[1].as_ref(), b"bc");
-        assert_eq!(chunks[2].as_ref(), b"de");
-        assert!(chunks[3].is_empty());
-        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
-    }
+        let err = fixture
+            .dataset
+            .read_blob_ranges("blobs")
+            .unwrap()
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+        assert!(err.to_string().contains("requires requests"));
 
-    #[tokio::test]
-    async fn test_blob_file_read_range_rejects_out_of_bounds() {
-        let (store, _) = recording_range_store(Bytes::from_static(b"abcdef"));
-        let path = Path::from("blobs/test.bin");
-        let blob = BlobFile::new_packed(store, path, 0, 4);
+        let err = fixture
+            .dataset
+            .read_blob_ranges("blobs")
+            .unwrap()
+            .with_row_indices([BlobRangeRequest::new(0, u64::MAX, 1)])
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+        assert!(err.to_string().contains("offset + length overflowed"));
 
-        let err = blob.read_range(1..5).await.unwrap_err();
+        let err = fixture
+            .dataset
+            .read_blob_ranges("blobs")
+            .unwrap()
+            .with_row_indices([BlobRangeRequest::new(0, 0, u64::MAX)])
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
         assert!(err.to_string().contains("exceeds blob size"));
+        assert!(err.to_string().contains("request 0"));
     }
 
     #[tokio::test]
-    async fn test_blob_files_share_source_and_coalesce() {
-        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
-        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
-        let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None);
-        let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None);
+    async fn test_planned_blob_range_avoids_whole_value_read_amplification() {
+        const VALUE_SIZE: u64 = 500 * 1024 * 1024;
+        const WINDOW_SIZE: u64 = 100 * 1024;
+
+        let (store, inner) = recording_range_store(Bytes::from(vec![7; WINDOW_SIZE as usize]));
+        let source = Arc::new(BlobSource::new(store, Path::from("blobs/large.bin")));
+        source
+            .file_size
+            .set(std::num::NonZeroU64::new(VALUE_SIZE).unwrap());
+        let entries = vec![BlobEntry {
+            selection_index: 0,
+            row_address: 10,
+            file: BlobFile::with_source(source, 0, VALUE_SIZE, BlobKind::Dedicated, None),
+        }];
+        let requested = [BlobReadRange::new(0, WINDOW_SIZE)];
+        let mut plans = plan_blob_read_plans(entries, Some(&requested)).unwrap();
+        let results = execute_blob_read_plan(
+            plans.pop().unwrap(),
+            Arc::new(ReadBlobsExecution::new(None)),
+        )
+        .await
+        .unwrap();
 
-        let (data1, data2) = tokio::join!(blob1.read(), blob2.read());
-        assert_eq!(data1.unwrap().as_ref(), b"bcd");
-        assert_eq!(data2.unwrap().as_ref(), b"efg");
-        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].data.len(), WINDOW_SIZE as usize);
+        let physical_ranges = inner.requested_blob_ranges();
+        assert_eq!(physical_ranges, vec![0..WINDOW_SIZE]);
+        assert_eq!(
+            physical_ranges
+                .iter()
+                .map(|range| range.end - range.start)
+                .sum::<u64>(),
+            WINDOW_SIZE
+        );
+        assert!(!physical_ranges.contains(&(0..VALUE_SIZE)));
     }
 
     #[tokio::test]
-    async fn test_read_blobs_plan_preserves_order_and_coalesces() {
-        let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
-        let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
-        let entries = vec![
-            BlobEntry {
-                selection_index: 0,
-                row_address: 10,
-                file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None),
-            },
-            BlobEntry {
-                selection_index: 1,
-                row_address: 11,
-                file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None),
-            },
-        ];
-        let execution = Arc::new(ReadBlobsExecution::new(None));
-        let blobs = try_join_all(
-            plan_blob_read_plans(entries)
-                .into_iter()
-                .map(|plan| execute_blob_read_plan(plan, execution.clone())),
+    async fn test_read_blob_ranges_avoids_whole_value_read_amplification_end_to_end() {
+        const VALUE_SIZE: usize = 8 * 1024 * 1024;
+        const WINDOW_SIZE: u64 = 100 * 1024;
+
+        let test_dir = TempStrDir::default();
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_bytes(vec![0xA5; VALUE_SIZE]).unwrap();
+        let mut blob_field = blob_field("blob", false);
+        let mut metadata = blob_field.metadata().clone();
+        metadata.insert(
+            BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY.to_string(),
+            "1".to_string(),
+        );
+        blob_field = blob_field.with_metadata(metadata);
+        let schema = Arc::new(Schema::new(vec![blob_field]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(blob_builder.finish().unwrap())],
         )
-        .await
         .unwrap();
-        let mut blobs = blobs.into_iter().flatten().collect::<Vec<_>>();
-        blobs.sort_by_key(|blob| blob.selection_index);
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
 
-        assert_eq!(blobs.len(), 2);
-        assert_eq!(blobs[0].row_address, 10);
-        assert_eq!(blobs[0].data.as_ref(), b"efg");
-        assert_eq!(blobs[1].row_address, 11);
-        assert_eq!(blobs[1].data.as_ref(), b"bcd");
-        assert_eq!(inner.requested_blob_ranges(), vec![1..7]);
+        let _ = dataset.object_store.io_stats_incremental();
+        let results = dataset
+            .read_blob_ranges("blob")
+            .unwrap()
+            .with_row_indices([BlobRangeRequest::new(0, 4 * 1024 * 1024, WINDOW_SIZE)])
+            .execute()
+            .await
+            .unwrap();
+        let stats = dataset.object_store.io_stats_incremental();
+
+        assert_eq!(results.len(), 1);
+        let data = results[0].data.as_ref().unwrap();
+        assert_eq!(data.len(), WINDOW_SIZE as usize);
+        assert!(data.iter().all(|byte| *byte == 0xA5));
+        let blob_payload_ranges = stats
+            .requests
+            .iter()
+            .filter(|request| request.path.as_ref().ends_with(".blob"))
+            .filter_map(|request| request.range.clone())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            blob_payload_ranges,
+            vec![4 * 1024 * 1024..4 * 1024 * 1024 + WINDOW_SIZE]
+        );
     }
 
     #[tokio::test]
@@ -4015,7 +6299,8 @@ mod tests {
         ];
         let execution = Arc::new(ReadBlobsExecution::new(None));
         let mut stream: super::ReadBlobsStream = futures::stream::iter(
-            plan_blob_read_plans(entries)
+            plan_blob_read_plans(entries, None)
+                .unwrap()
                 .into_iter()
                 .map(move |plan| execute_blob_read_plan(plan, execution.clone())),
         )
@@ -4036,16 +6321,16 @@ mod tests {
             .unwrap()
             .unwrap();
         assert_eq!(first.row_address, 11);
-        assert_eq!(first.data.as_ref(), b"uvw");
+        assert_eq!(first.data.as_deref(), Some(b"uvw".as_slice()));
 
-        slow_gate.notify_one();
+        slow_gate.add_permits(1);
         let second = tokio::time::timeout(Duration::from_secs(1), stream.next())
             .await
             .unwrap()
             .unwrap()
             .unwrap();
         assert_eq!(second.row_address, 10);
-        assert_eq!(second.data.as_ref(), b"abc");
+        assert_eq!(second.data.as_deref(), Some(b"abc".as_slice()));
     }
 
     #[tokio::test]
@@ -4059,9 +6344,10 @@ mod tests {
             .unwrap();
 
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Inline);
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Inline);
         assert_eq!(
-            blobs[0].read().await.unwrap().as_ref(),
+            blob.read().await.unwrap().as_ref(),
             fixture.expected.as_slice()
         );
     }
@@ -4079,9 +6365,10 @@ mod tests {
             .unwrap();
 
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Packed);
         assert_eq!(
-            blobs[0].read().await.unwrap().as_ref(),
+            blob.read().await.unwrap().as_ref(),
             fixture.expected.as_slice()
         );
     }
@@ -4097,9 +6384,10 @@ mod tests {
             .unwrap();
 
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Dedicated);
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Dedicated);
         assert_eq!(
-            blobs[0].read().await.unwrap().as_ref(),
+            blob.read().await.unwrap().as_ref(),
             fixture.expected.as_slice()
         );
     }
@@ -4117,9 +6405,10 @@ mod tests {
             .unwrap();
 
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Packed);
         assert_eq!(
-            blobs[0].read().await.unwrap().as_ref(),
+            blob.read().await.unwrap().as_ref(),
             fixture.expected.as_slice()
         );
     }
@@ -4240,7 +6529,10 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"outside");
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"outside"
+        );
     }
 
     #[tokio::test]
@@ -4303,7 +6595,10 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"mapped");
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"mapped"
+        );
     }
 
     #[tokio::test]
@@ -4388,8 +6683,9 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Inline);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"inline");
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Inline);
+        assert_eq!(blob.read().await.unwrap().as_ref(), b"inline");
     }
 
     #[tokio::test]
@@ -4442,8 +6738,9 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Packed);
+        assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice());
     }
 
     #[tokio::test]
@@ -4486,8 +6783,9 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Packed);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Packed);
+        assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice());
     }
 
     #[tokio::test]
@@ -4540,8 +6838,9 @@ mod tests {
 
         let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
         assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Dedicated);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+        let blob = blobs[0].as_ref().unwrap();
+        assert_eq!(blob.kind(), BlobKind::Dedicated);
+        assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice());
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs
index 65928038cea..d8f69e7d36a 100644
--- a/rust/lance/src/dataset/cleanup.rs
+++ b/rust/lance/src/dataset/cleanup.rs
@@ -58,8 +58,8 @@ use lance_table::{
         manifest::{read_manifest, read_manifest_indexes},
     },
 };
+use object_store::ObjectMeta;
 use object_store::path::Path;
-use object_store::{Error as ObjectStoreError, ObjectMeta};
 use std::fmt::Debug;
 use std::{
     collections::{HashMap, HashSet},
@@ -625,16 +625,7 @@ impl<'a> CleanupTask<'a> {
         let verification_threshold = utc_now()
             - TimeDelta::try_days(UNVERIFIED_THRESHOLD_DAYS).expect("TimeDelta::try_days");
 
-        let is_not_found_err = |e: &Error| {
-            matches!(
-                e,
-                Error::IO { source,.. }
-                    if source
-                      .downcast_ref::<ObjectStoreError>()
-                      .map(|os_err| matches!(os_err, ObjectStoreError::NotFound {.. }))
-                      .unwrap_or(false)
-            )
-        };
+        let is_not_found_err = |e: &Error| matches!(e, Error::NotFound { .. });
         // Build stream for a managed subtree
         let build_listing_stream = |dir: Path| {
             let inspection_ref = &inspection;
@@ -2987,10 +2978,8 @@ mod tests {
     async fn cleanup_before_ts_and_retain_n_recent_versions() {
         let fixture = MockDatasetFixture::try_new().unwrap();
         fixture.create_some_data().await.unwrap();
-        let mut time = 1i64;
-        for _ in 0..4 {
+        for time in (1i64..).take(4) {
             MockClock::set_system_time(TimeDelta::try_days(time).unwrap().to_std().unwrap());
-            time += 1i64;
             fixture.overwrite_some_data().await.unwrap();
         }
 
diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs
index 848add7e4a8..2214822ed90 100644
--- a/rust/lance/src/dataset/files.rs
+++ b/rust/lance/src/dataset/files.rs
@@ -1036,6 +1036,7 @@ mod tests {
                 // No base_id -> falls back to the dataset base_uri.
                 mk_file("c.lance", None),
             ],
+            overlays: vec![],
             // Deletion files also carry a base_id when they originate from a
             // shallow clone, and must resolve against base_paths too.
             deletion_file: Some(DeletionFile {
diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs
index 175b1d7d2de..f6bc94567a0 100644
--- a/rust/lance/src/dataset/fragment.rs
+++ b/rust/lance/src/dataset/fragment.rs
@@ -22,6 +22,7 @@ use datafusion::logical_expr::Expr;
 use datafusion::scalar::ScalarValue;
 use futures::future::{BoxFuture, try_join_all};
 use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, join, stream};
+use lance_arrow::json::{convert_json_columns, has_json_fields, is_arrow_json_field};
 use lance_arrow::{RecordBatchExt, SchemaExt};
 use lance_core::datatypes::{OnMissing, OnTypeMismatch, SchemaCompareOptions};
 use lance_core::utils::address::RowAddress;
@@ -64,6 +65,9 @@ use super::updater::Updater;
 use super::{NewColumnTransform, WriteParams, schema_evolution};
 use crate::dataset::Dataset;
 use crate::dataset::fragment::session::FragmentSession;
+use crate::dataset::overlay::{
+    OverlayReadPlanner, merge_overlay_batch, plan_overlays, resolve_overlays,
+};
 use crate::io::deletion::read_dataset_deletion_file;
 
 /// Result of [`FileFragment::update_columns_with_offsets`]: updated fragment metadata, modified field ids,
@@ -648,7 +652,7 @@ impl GenericFileReader for NullReader {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct FragReadConfig {
     // Add the row id column
     pub with_row_id: bool,
@@ -948,6 +952,19 @@ impl FileFragment {
             Arc::new(self.metadata.clone()),
         )?;
 
+        // Plan overlay resolution from coverage metadata (no files opened here); the
+        // readers are opened lazily on read, pruned to the rows each read touches.
+        if !self.metadata.overlays.is_empty() {
+            let planner = plan_overlays(self, projection)?;
+            if !planner.is_empty() {
+                reader.overlay = Some(OverlayReadState {
+                    planner: Arc::new(planner),
+                    fragment: Arc::new(self.clone()),
+                    read_config: Arc::new(read_config.clone()),
+                });
+            }
+        }
+
         if read_config.with_row_id {
             reader.with_row_id();
         }
@@ -985,7 +1002,7 @@ impl FileFragment {
         selected_columns.saturating_mul(4) < total_columns
     }
 
-    async fn open_reader(
+    pub(super) async fn open_reader(
         &self,
         data_file: &DataFile,
         projection: Option<&Schema>,
@@ -1603,7 +1620,7 @@ impl FileFragment {
     }
 
     /// Get the file metadata for this fragment, using the cache if available.
-    async fn get_file_metadata(
+    pub async fn get_file_metadata(
         &self,
         file_scheduler: &FileScheduler,
     ) -> Result<Arc<CachedFileMetadata>> {
@@ -1620,7 +1637,7 @@ impl FileFragment {
         Ok(file_metadata)
     }
 
-    async fn get_file_metadata_index(
+    pub async fn get_file_metadata_index(
         &self,
         file_scheduler: &FileScheduler,
         known_schema: Option<(Arc<Schema>, u64)>,
@@ -1906,6 +1923,17 @@ impl FileFragment {
             )
             .await?;
         // Hash join: rows matched on the right-hand stream rewrite columns; track physical offsets via `_rowaddr`.
+        // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) in the right stream
+        // so they match the physical storage format read from the fragment's left batch.
+        let right_stream: Box<dyn RecordBatchReader + Send> = if right_schema
+            .fields()
+            .iter()
+            .any(|f| is_arrow_json_field(f) || has_json_fields(f))
+        {
+            Box::new(JsonConvertingReader::new(right_stream))
+        } else {
+            right_stream
+        };
         let joiner = Arc::new(HashJoiner::try_new(right_stream, right_on).await?);
         let mut matched_offsets = RoaringBitmap::new();
         let frag_id_u32 = u32::try_from(self.metadata.id).map_err(|_| {
@@ -2242,6 +2270,23 @@ pub struct FragmentReader {
 
     // total number of physical rows in the fragment (all rows, ignoring deletions)
     num_physical_rows: usize,
+
+    /// Read-time state for resolving data overlay files: the coverage plan plus
+    /// what is needed to open overlay readers. `None` when the fragment has no
+    /// overlays. Overlays are merged into base batches (by `offset_in_frag`) before
+    /// deletion filtering, opening only the files each read's rows touch.
+    overlay: Option<OverlayReadState>,
+}
+
+/// What [`FragmentReader`] needs to resolve overlays at read time: the coverage
+/// plan (from metadata, cheap to build), and the fragment + config needed to open
+/// overlay readers once the read's rows — and therefore which files it touches —
+/// are known. All `Arc` so cloning a reader stays cheap.
+#[derive(Clone, Debug)]
+struct OverlayReadState {
+    planner: Arc<OverlayReadPlanner>,
+    fragment: Arc<FileFragment>,
+    read_config: Arc<FragReadConfig>,
 }
 
 // Custom clone impl needed because it is not easy to clone Box<dyn GenericFileReader>
@@ -2270,6 +2315,7 @@ impl Clone for FragmentReader {
             created_at_sequence: self.created_at_sequence.clone(),
             num_rows: self.num_rows,
             num_physical_rows: self.num_physical_rows,
+            overlay: self.overlay.clone(),
         }
     }
 }
@@ -2337,6 +2383,7 @@ impl FragmentReader {
             created_at_sequence: None,
             num_rows,
             num_physical_rows,
+            overlay: None,
         })
     }
 
@@ -2594,6 +2641,77 @@ impl FragmentReader {
         Ok(result.project_by_schema(&output_schema)?)
     }
 
+    /// Merge data overlay values onto a stream of base batches.
+    ///
+    /// Runs on physical rows in read order, *before* deletion filtering, so each
+    /// row can be addressed by its position in the fragment (its `offset_in_frag`,
+    /// derived from `params`) and deletions take precedence naturally: an overlay
+    /// value for a deleted row is dropped along with the row downstream. A no-op
+    /// when the fragment has no overlays.
+    ///
+    /// The read's `offset_in_frag` values are known from `params` up front, so
+    /// overlays are resolved here to just the files this read's rows touch — an
+    /// overlay whose cells fall outside the read is not opened at all. Within each
+    /// batch, the overlay reads (only the values that batch needs) are then issued
+    /// concurrently with the base read rather than after it.
+    async fn merge_overlays(
+        &self,
+        merged: ReadBatchTaskStream,
+        params: &ReadBatchParams,
+        total_num_rows: u32,
+    ) -> Result<ReadBatchTaskStream> {
+        let Some(overlay) = &self.overlay else {
+            return Ok(merged);
+        };
+        // The offset_in_frag of every row this read will return, materialized once.
+        // Cost is one u32 per output row (a whole-fragment scan is 4 bytes/row), and
+        // it lets us both prune overlays to the read and slice each batch's offsets
+        // below without reading any data. Only paid when the fragment has overlays.
+        //
+        // TODO(overlay perf): this could be avoided by teaching `ReadBatchParams` to
+        // yield a coverage bitmap directly (for pruning) and to slice per batch (for
+        // the routing below), or by moving `ReadBatchParams` to a roaring bitmap
+        // wholesale — a larger refactor tracked separately.
+        let offsets_in_frag: Arc<Vec<u32>> =
+            Arc::new(params.to_offsets_total(total_num_rows).values().to_vec());
+
+        // Open only the overlay readers this read touches (pruned by row selection).
+        let plans = resolve_overlays(
+            &overlay.planner,
+            &offsets_in_frag,
+            &overlay.fragment,
+            &overlay.read_config,
+        )
+        .await?;
+        if plans.is_empty() {
+            return Ok(merged);
+        }
+        let plans = Arc::new(plans);
+
+        // Batches arrive in physical read order, so a running total of the rows seen
+        // so far gives each batch its starting offset_in_batch into `offsets_in_frag`.
+        let mut rows_seen = 0usize;
+        let stream = merged
+            .map(move |task| {
+                let num_rows = task.num_rows;
+                let start = rows_seen;
+                rows_seen += num_rows as usize;
+                let offsets_in_frag = offsets_in_frag.clone();
+                let plans = plans.clone();
+                let inner = task.task;
+                ReadBatchTask {
+                    num_rows,
+                    task: async move {
+                        let batch_offsets = &offsets_in_frag[start..start + num_rows as usize];
+                        merge_overlay_batch(inner, batch_offsets, &plans).await
+                    }
+                    .boxed(),
+                }
+            })
+            .boxed();
+        Ok(stream)
+    }
+
     async fn new_read_impl<'a, F>(
         &'a self,
         params: ReadBatchParams,
@@ -2661,6 +2779,8 @@ impl FragmentReader {
             lance_table::utils::stream::merge_streams(read_streams)
         };
 
+        let merged = self.merge_overlays(merged, &params, total_num_rows).await?;
+
         // Add the row id column (if needed) and delete rows (if a deletion
         // vector is present).
         let config = RowIdAndDeletesConfig {
@@ -2831,6 +2951,11 @@ impl FragmentReader {
             lance_table::utils::stream::merge_streams(read_streams)
         };
 
+        let params = ReadBatchParams::Ranges(ranges);
+        let merged_stream = self
+            .merge_overlays(merged_stream, &params, total_num_rows)
+            .await?;
+
         // Add the row id column (if needed) and delete rows (if a deletion
         // vector is present).
         let config = RowIdAndDeletesConfig {
@@ -2843,7 +2968,7 @@ impl FragmentReader {
             with_row_created_at_version: self.with_row_created_at_version,
             last_updated_at_sequence: self.last_updated_at_sequence.clone(),
             created_at_sequence: self.created_at_sequence.clone(),
-            params: ReadBatchParams::Ranges(ranges),
+            params,
             total_num_rows,
         };
         let output_schema = Arc::new(self.output_schema.clone());
@@ -2946,6 +3071,59 @@ impl FragmentReader {
     }
 }
 
+/// A wrapper around a `RecordBatchReader` that converts Arrow JSON columns
+/// (Utf8/LargeUtf8 with `arrow.json` extension) to Lance JSON columns
+/// (LargeBinary with `lance.json` extension / JSONB format).
+///
+/// This is needed when user-provided data contains Arrow JSON fields but the
+/// dataset stores them in Lance's JSONB binary format.
+struct JsonConvertingReader {
+    inner: Box<dyn RecordBatchReader + Send>,
+    schema: arrow_schema::SchemaRef,
+}
+
+impl JsonConvertingReader {
+    fn new(inner: Box<dyn RecordBatchReader + Send>) -> Self {
+        use lance_arrow::json::arrow_json_to_lance_json;
+
+        // Build the converted schema (Arrow JSON fields → Lance JSON fields)
+        let orig_schema = inner.schema();
+        let new_fields: Vec<arrow_schema::FieldRef> = orig_schema
+            .fields()
+            .iter()
+            .map(|f| {
+                if is_arrow_json_field(f) || has_json_fields(f) {
+                    Arc::new(arrow_json_to_lance_json(f))
+                } else {
+                    Arc::clone(f)
+                }
+            })
+            .collect();
+        let schema = Arc::new(arrow_schema::Schema::new_with_metadata(
+            new_fields,
+            orig_schema.metadata().clone(),
+        ));
+
+        Self { inner, schema }
+    }
+}
+
+impl Iterator for JsonConvertingReader {
+    type Item = std::result::Result<RecordBatch, arrow_schema::ArrowError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner
+            .next()
+            .map(|result| result.and_then(|batch| convert_json_columns(&batch)))
+    }
+}
+
+impl RecordBatchReader for JsonConvertingReader {
+    fn schema(&self) -> arrow_schema::SchemaRef {
+        self.schema.clone()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use arrow_arith::numeric::mul;
@@ -3039,6 +3217,1519 @@ mod tests {
         Dataset::open(test_uri).await.unwrap()
     }
 
+    /// End-to-end tests for reading data overlay files (OSS-1324): overlays are
+    /// written, committed via the `DataOverlay` transaction, and then resolved on
+    /// the `take` and scan read paths.
+    mod overlay_read {
+        use std::sync::Arc;
+
+        use arrow_array::{
+            Array, ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StructArray, UInt64Array,
+        };
+        use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
+        use lance_core::datatypes::Schema;
+        use lance_file::version::LanceFileVersion;
+        use lance_file::writer::{FileWriter, FileWriterOptions};
+        use lance_io::utils::CachedFileSize;
+        use lance_table::format::DataFile;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use object_store::path::Path;
+        use roaring::RoaringBitmap;
+        use rstest::rstest;
+
+        use crate::dataset::transaction::{DataOverlayGroup, Operation};
+        use crate::dataset::{Dataset, WriteDestination, WriteParams};
+
+        fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+            RoaringBitmap::from_iter(offsets)
+        }
+
+        fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+            Arc::new(Int32Array::from_iter(values))
+        }
+
+        /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1)
+        /// = id * 10, written 6 rows per file (fragments 0 and 1).
+        ///
+        /// Uses an in-memory store so the test can write overlay files with a
+        /// store-relative `data/<name>.lance` path and commit against the returned
+        /// dataset directly.
+        async fn create_base_dataset(version: LanceFileVersion) -> Dataset {
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..12)),
+                    Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap()
+        }
+
+        /// Write an overlay file covering `fields` (dataset field ids) of
+        /// `fragment_id` with the given coverage and per-field value columns, then
+        /// commit it as a `DataOverlay` transaction. `name` makes the file unique.
+        #[allow(clippy::too_many_arguments)]
+        async fn commit_overlay(
+            dataset: Dataset,
+            name: &str,
+            fragment_id: u64,
+            fields: &[i32],
+            coverage: OverlayCoverage,
+            columns: Vec<ArrayRef>,
+            version: LanceFileVersion,
+        ) -> Dataset {
+            let read_version = dataset.version().version;
+            let overlay_schema = dataset.schema().project_by_ids(fields, true);
+
+            let filename = format!("{name}.lance");
+            let path = Path::from(format!("data/{filename}"));
+            let obj_writer = dataset.object_store.create(&path).await.unwrap();
+            let mut writer = FileWriter::try_new(
+                obj_writer,
+                overlay_schema,
+                FileWriterOptions {
+                    format_version: Some(version),
+                    ..Default::default()
+                },
+            )
+            .unwrap();
+            let (major, minor) = writer.version().to_numbers();
+            for (column_index, array) in columns.into_iter().enumerate() {
+                writer.write_column(column_index, array).await.unwrap();
+            }
+            let summary = writer.finish().await.unwrap();
+
+            let mut data_file = DataFile::new_unstarted(filename, major, minor);
+            data_file.fields = writer
+                .field_id_to_column_indices()
+                .iter()
+                .map(|(field_id, _)| *field_id as i32)
+                .collect::<Vec<_>>()
+                .into();
+            data_file.column_indices = writer
+                .field_id_to_column_indices()
+                .iter()
+                .map(|(_, column_index)| *column_index as i32)
+                .collect::<Vec<_>>()
+                .into();
+            data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+            let overlay = DataOverlayFile {
+                data_file,
+                coverage,
+                committed_version: 0,
+            };
+            Dataset::commit(
+                WriteDestination::Dataset(Arc::new(dataset)),
+                Operation::DataOverlay {
+                    groups: vec![DataOverlayGroup {
+                        fragment_id,
+                        overlays: vec![overlay],
+                    }],
+                },
+                Some(read_version),
+                None,
+                None,
+                Arc::new(Default::default()),
+                false,
+            )
+            .await
+            .unwrap()
+        }
+
+        fn full_schema(dataset: &Dataset) -> Schema {
+            dataset.schema().clone()
+        }
+
+        fn col(batch: &RecordBatch, name: &str) -> Int32Array {
+            let idx = batch.schema().index_of(name).unwrap();
+            batch
+                .column(idx)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .clone()
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_covered_and_uncovered(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay fragment 0's `val` at physical offsets {1, 4}.
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag
+                .take(&[0, 1, 2, 4], &full_schema(&dataset))
+                .await
+                .unwrap();
+            // Offsets 1 and 4 take overlay values; 0 and 2 fall through to base.
+            assert_eq!(col(&batch, "val").values(), &[0, 111, 20, 444]);
+            // The unrelated `id` column is untouched.
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 4]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_newest_overlay_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+            // A newer overlay (later commit -> higher committed_version) re-covers
+            // offset 1.
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 4], &full_schema(&dataset)).await.unwrap();
+            // Offset 1 -> newest overlay (999); offset 4 -> only older covers it.
+            assert_eq!(col(&batch, "val").values(), &[999, 444]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_per_field_coverage(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Sparse overlay: `id` covers {2}, `val` covers {2, 3} — different
+            // offset sets and therefore unequal-length value columns.
+            let dataset = commit_overlay(
+                dataset,
+                "sparse",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([2]), bitmap([2, 3])]),
+                vec![i32_array([Some(777)]), i32_array([Some(220), Some(330)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap();
+            // id: offset 2 covered (777), offset 3 falls through (3).
+            assert_eq!(col(&batch, "id").values(), &[777, 3]);
+            // val: both offsets covered (220, 330).
+            assert_eq!(col(&batch, "val").values(), &[220, 330]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_null_override(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "nullov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([None])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[0, 1], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            // Offset 0 is covered with a NULL value -> resolves to NULL; offset 1
+            // falls through to the base value.
+            assert!(val.is_null(0));
+            assert_eq!(val.value(1), 10);
+        }
+
+        /// Overlays interact correctly with NULL *base* cells (distinct from a NULL
+        /// overlay value): a covered row whose base value is NULL is overridden to the
+        /// overlay's non-null value, while an uncovered NULL base cell falls through
+        /// and stays NULL.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_null_base_cell(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    // `val` is NULL at offsets 1 and 3.
+                    Arc::new(Int32Array::from_iter([
+                        Some(0),
+                        None,
+                        Some(20),
+                        None,
+                        Some(40),
+                        Some(50),
+                    ])),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Cover offset 1 (NULL base) and offset 4 (non-null base); leave offset
+            // 3's NULL base uncovered.
+            let dataset = commit_overlay(
+                dataset,
+                "nullbase",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 3, 4], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            // Offset 1: NULL base overridden to 111. Offset 3: uncovered NULL base
+            // stays NULL. Offset 4: non-null base overridden to 444.
+            assert_eq!(val.value(0), 111);
+            assert!(val.is_null(1));
+            assert_eq!(val.value(2), 444);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_on_deleted_row_is_inert(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let mut dataset = create_base_dataset(version).await;
+            // Delete global row 1 (fragment 0, physical offset 1).
+            dataset.delete("id = 1").await.unwrap();
+            // Overlay covers the deleted offset 1 and the live offset 4.
+            let dataset = commit_overlay(
+                dataset,
+                "delov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            // Scan fragment 0: row 1 is gone, and offset 4's overlay value survives
+            // even though the deletion shifts logical positions — coverage is keyed
+            // by physical offset.
+            let frag = dataset.get_fragment(0).unwrap();
+            let mut scanner = frag.scan();
+            let batch = scanner
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 2, 3, 4, 5]);
+            assert_eq!(col(&batch, "val").values(), &[0, 20, 30, 444, 50]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_multi_fragment_overlays(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay fragment 0 at offset 0 and fragment 1 at offset 0 (global
+            // row 6). Each fragment's coverage is independent.
+            let dataset = commit_overlay(
+                dataset,
+                "frag0",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag1",
+                1,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(6000)])],
+                version,
+            )
+            .await;
+
+            let batch = dataset
+                .scan()
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(batch.num_rows(), 12);
+            let expected: Vec<i32> = (0..12)
+                .map(|i| match i {
+                    0 => 1000,
+                    6 => 6000,
+                    other => other * 10,
+                })
+                .collect();
+            assert_eq!(col(&batch, "val").values(), &expected);
+        }
+
+        /// A `take` of a few rows must read only the overlay values those rows
+        /// touch — not the whole column. Uses v2.1 (which slices pages on read) and
+        /// an incompressible, all-covering overlay, so reading the full column would
+        /// be far more bytes than reading a couple of values. This is the regression
+        /// guard for the lazy, value-pushdown overlay read.
+        #[tokio::test]
+        async fn test_take_reads_only_needed_overlay_values() {
+            let version = LanceFileVersion::V2_1;
+            const N: usize = 100_000;
+
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let base = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..N as i32)),
+                    Arc::new(Int32Array::from_iter_values((0..N as i32).map(|v| v * 10))),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: N,
+                max_rows_per_group: N,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(base)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `val` over ALL N offsets with incompressible values, so the
+            // value column is ~N*4 bytes on disk.
+            let values: Vec<i32> = (0..N as u64)
+                .map(|i| {
+                    let mut x = i;
+                    x ^= x >> 33;
+                    x = x.wrapping_mul(0xff51_afd7_ed55_8ccd);
+                    x ^= x >> 33;
+                    x as i32
+                })
+                .collect();
+            let dataset = commit_overlay(
+                dataset,
+                "big",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap(0..N as u32)),
+                vec![Arc::new(Int32Array::from(values.clone())) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // Measure only the reads that resolve the take.
+            dataset.object_store.io_stats_incremental();
+            let batch = frag.take(&[0, 1], &val_only).await.unwrap();
+            let io = dataset.object_store.io_stats_incremental();
+
+            // The overlay's `val` column alone is N*4 bytes; resolving two adjacent
+            // offsets must read only a small fraction of it.
+            let full_column_bytes = (N * std::mem::size_of::<i32>()) as u64;
+            assert!(
+                io.read_bytes > 0 && io.read_bytes < full_column_bytes / 4,
+                "take read {} bytes; expected far less than the {}-byte overlay \
+                 column (a take must not read the whole value column)",
+                io.read_bytes,
+                full_column_bytes,
+            );
+
+            // ...and it still resolves correctly.
+            let val = col(&batch, "val");
+            assert_eq!(val.value(0), values[0]);
+            assert_eq!(val.value(1), values[1]);
+        }
+
+        /// Row-selection pruning: an overlay whose coverage is disjoint from the
+        /// requested rows must not be opened at all. Proven by deleting the overlay's
+        /// data file — a `take` that misses its coverage still succeeds (the file is
+        /// never touched), while a `take` that hits it then fails because the file is
+        /// genuinely needed.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_prunes_overlays_outside_row_selection(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay on fragment 0 (offsets 0..6) covering only offset_in_frag 5.
+            let dataset = commit_overlay(
+                dataset,
+                "miss",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([5])),
+                vec![i32_array([Some(5000)])],
+                version,
+            )
+            .await;
+
+            // Delete the overlay's data file: opening it now fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/miss.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // A take that misses the overlay's coverage must not open it, so it
+            // succeeds and returns base values (val = offset * 10).
+            let batch = frag.take(&[0, 1], &val_only).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[0, 10]);
+
+            // A take that hits the coverage does need the file, so it now fails with
+            // a not-found error naming the missing overlay file.
+            let err = frag.take(&[5], &val_only).await.unwrap_err();
+            let message = format!("{err:?}");
+            assert!(
+                err.is_not_found() && message.contains("miss.lance"),
+                "take hitting the overlay's coverage should fail with a not-found error \
+                 for its missing file, got: {message}",
+            );
+        }
+
+        /// The overlay merge runs before `wrap_with_row_id_and_delete`, so the
+        /// `_rowid` system column must coexist with overlay-resolved data columns:
+        /// the row ids are unaffected by the merge and the overlay value still wins.
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_with_row_id_alongside_overlay(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "rowidov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag
+                .scan()
+                .with_row_id()
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            // Overlay value resolves...
+            assert_eq!(col(&batch, "val").values()[0], 1000);
+            assert_eq!(&col(&batch, "val").values()[1..], &[10, 20, 30, 40, 50]);
+            // ...and the row ids for fragment 0 are the untouched physical offsets.
+            let row_ids = batch
+                .column(batch.schema().index_of("_rowid").unwrap())
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .unwrap();
+            assert_eq!(row_ids.values(), &[0, 1, 2, 3, 4, 5]);
+        }
+
+        /// When the newest overlay covers every requested offset, an older overlay
+        /// in the same plan needs zero values and its value column must not be read
+        /// (the empty-input branch of `fetch_overlay_values`). The result still
+        /// resolves to the newest overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_older_overlay_contributes_no_values(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Older covers {1, 4}; newer re-covers {1}. A take of only offset 1
+            // routes entirely to the newer overlay, leaving the older one with no
+            // values to fetch even though it is part of the field's plan.
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[999]);
+        }
+
+        /// A newest overlay whose value is NULL must shadow an older overlay's
+        /// non-null value at the same offset — the merge resolves to NULL, it does
+        /// not fall back to the older overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_newest_null_shadows_older(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(111)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "newer_null",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([None])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            assert!(val.is_null(0), "newest NULL must win over older 111");
+        }
+
+        /// Newest-wins is resolved independently per field across multiple sparse
+        /// overlays: for the same offset, `id` can resolve to one overlay while
+        /// `val` resolves to the other, depending on which overlay newly covers
+        /// that field at that offset.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_multi_sparse_per_field_newest_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Older: id covers {3}, val covers {2}.
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([3]), bitmap([2])]),
+                vec![i32_array([Some(7773)]), i32_array([Some(2772)])],
+                version,
+            )
+            .await;
+            // Newer: id covers {2}, val covers {3} — the mirror image.
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([2]), bitmap([3])]),
+                vec![i32_array([Some(9992)]), i32_array([Some(9993)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap();
+            // id: offset 2 -> newer (9992), offset 3 -> older (7773).
+            assert_eq!(col(&batch, "id").values(), &[9992, 7773]);
+            // val: offset 2 -> older (2772), offset 3 -> newer (9993).
+            assert_eq!(col(&batch, "val").values(), &[2772, 9993]);
+        }
+
+        /// A fragment with an overlay plan, but a take that touches only uncovered
+        /// offsets, must fall entirely through to the base values (the
+        /// `!routing.any_overlay` early-return with a plan present).
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_plan_present_all_offsets_uncovered(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            // None of {0, 2, 5} are covered: the plan exists but contributes nothing.
+            let batch = frag.take(&[0, 2, 5], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[0, 20, 50]);
+            assert_eq!(col(&batch, "id").values(), &[0, 2, 5]);
+        }
+
+        /// A dataset-level `take` spanning multiple fragments, each with its own
+        /// overlay, routes every global row index to the right fragment's overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_dataset_take_multi_fragment_overlays(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag0",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag1",
+                1,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(6000)])],
+                version,
+            )
+            .await;
+
+            // Global rows 0 and 6 are the overlaid offset-0 rows of fragments 0 and
+            // 1; rows 1 and 7 fall through to base.
+            let batch = dataset
+                .take(&[0, 1, 6, 7], full_schema(&dataset))
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 6, 7]);
+            assert_eq!(col(&batch, "val").values(), &[1000, 10, 6000, 70]);
+        }
+
+        /// A scan whose read splits into multiple batches must slice
+        /// `offsets_in_frag` per batch correctly — the running `rows_seen`
+        /// accumulator in `merge_overlays` gives each batch its start. Every other
+        /// scan test uses single-batch fragments, so this is the only guard for the
+        /// cross-batch (`start > 0`) path.
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_multi_batch_overlay_slicing(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use futures::TryStreamExt;
+
+            // One fragment of 10 rows so the read can be chunked below.
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..10)),
+                    Arc::new(Int32Array::from_iter_values((0..10).map(|v| v * 10))),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 100,
+                max_rows_per_group: 100,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay one offset in each batch that batch_size 4 produces (batches
+            // [0,4), [4,8), [8,10)): offsets 1, 5, 9 with distinct values. A wrong
+            // per-batch slice would misalign these.
+            let dataset = commit_overlay(
+                dataset,
+                "multibatch",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 5, 9])),
+                vec![i32_array([Some(111), Some(555), Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let mut scanner = frag.scan();
+            scanner.batch_size(4).project(&["val"]).unwrap();
+            let batches: Vec<RecordBatch> = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect()
+                .await
+                .unwrap();
+            // Guard the guard: the read must actually span multiple batches, else
+            // this would not exercise the cross-batch slice at all.
+            assert!(
+                batches.len() > 1,
+                "expected a multi-batch scan, got {} batch(es)",
+                batches.len()
+            );
+
+            let merged =
+                arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
+            let expected: Vec<i32> = (0..10)
+                .map(|i| match i {
+                    1 => 111,
+                    5 => 555,
+                    9 => 999,
+                    other => other * 10,
+                })
+                .collect();
+            assert_eq!(col(&merged, "val").values(), &expected);
+        }
+
+        /// An empty selection must not trip over the overlay path: the plan exists
+        /// but there are no offsets to route, so the result is an empty batch.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_empty_selection(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(batch.num_rows(), 0);
+        }
+
+        /// Overlays resolve variable-width columns end-to-end, not just fixed-width
+        /// ones: the value column is fetched through the real file reader (a
+        /// different value-pushdown path than the fixed-width case) and assembled.
+        #[rstest]
+        #[tokio::test]
+        async fn test_string_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use arrow_array::StringArray;
+
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("name", DataType::Utf8, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e", "f"])),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `name` at offsets {1, 4}, one of the values NULL.
+            let dataset = commit_overlay(
+                dataset,
+                "strov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![Arc::new(StringArray::from(vec![Some("B"), None])) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[0, 1, 4], &full_schema(&dataset)).await.unwrap();
+            let name = batch
+                .column(batch.schema().index_of("name").unwrap())
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            assert_eq!(name.value(0), "a"); // falls through to base
+            assert_eq!(name.value(1), "B"); // overlay value
+            assert!(name.is_null(2)); // overlay NULL wins
+        }
+
+        /// Projection pruning must do NO IO to overlay files whose fields are not
+        /// projected. Proven the same way as row-selection pruning: delete the
+        /// overlay's data file, then read projecting only the *unrelated* `id`
+        /// column — it must succeed (the `val` overlay file is never opened), while
+        /// projecting the overlaid `val` column then fails because its file is gone.
+        #[rstest]
+        #[tokio::test]
+        async fn test_projection_prunes_overlay_files_no_io(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay covers `val` (field 1) only.
+            let dataset = commit_overlay(
+                dataset,
+                "valov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0, 1])),
+                vec![i32_array([Some(1000), Some(1010)])],
+                version,
+            )
+            .await;
+
+            // Delete the overlay's data file: opening it now fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/valov.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let id_only = dataset.schema().project_by_ids(&[0], true);
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // Projecting only `id` must not open the `val` overlay file, so it
+            // succeeds and returns untouched base values.
+            let batch = frag.take(&[0, 1], &id_only).await.unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1]);
+            // A scan projecting only `id` must likewise never touch the file.
+            let batch = frag
+                .scan()
+                .project(&["id"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 3, 4, 5]);
+
+            // Projecting the overlaid `val` column does need the file, so it fails
+            // with a not-found error naming the missing overlay file.
+            let err = frag.take(&[0], &val_only).await.unwrap_err();
+            let message = format!("{err:?}");
+            assert!(
+                err.is_not_found() && message.contains("valov.lance"),
+                "projecting the overlaid column should fail with a not-found error \
+                 for its missing file, got: {message}",
+            );
+        }
+
+        /// A top-level struct column resolves through overlays: the overlay stores
+        /// the struct's leaf columns (under V2_1 those are the only ids in
+        /// `data_file.fields`), and `plan_overlays` maps them back to the top-level
+        /// struct so the whole value is fetched and replaced as a unit.
+        #[rstest]
+        #[tokio::test]
+        async fn test_struct_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let struct_fields = Fields::from(vec![
+                ArrowField::new("x", DataType::Int32, true),
+                ArrowField::new("y", DataType::Int32, true),
+            ]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("info", DataType::Struct(struct_fields.clone()), true),
+            ]));
+            let info = Arc::new(StructArray::new(
+                struct_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), info],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay the whole `info` struct (top-level field id 1) at offset 2.
+            let overlay_info = Arc::new(StructArray::new(
+                struct_fields,
+                vec![
+                    Arc::new(Int32Array::from(vec![777])),
+                    Arc::new(Int32Array::from(vec![888])),
+                ],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "structov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay_info],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let info = batch
+                .column(batch.schema().index_of("info").unwrap())
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            let x = info
+                .column(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            let y = info
+                .column(1)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            // Offset 1 falls through to base {1, 100}; offset 2 takes the overlay.
+            assert_eq!(x.values(), &[1, 777]);
+            assert_eq!(y.values(), &[100, 888]);
+        }
+
+        /// A top-level list column resolves through overlays the same way — the
+        /// overlay's leaf (item) id maps back to the top-level list, and the whole
+        /// list value at a covered offset is replaced.
+        #[rstest]
+        #[tokio::test]
+        async fn test_list_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use arrow_array::ListArray;
+            use arrow_array::types::Int32Type;
+
+            let item = Arc::new(ArrowField::new("item", DataType::Int32, true));
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("tags", DataType::List(item.clone()), true),
+            ]));
+            let base_tags = ListArray::from_iter_primitive::<Int32Type, _, _>(
+                (0..6i32).map(|i| Some(vec![Some(i), Some(i * 10)])),
+            );
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(base_tags),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `tags` (top-level field id 1) at offset 2 with a new list.
+            let overlay_tags =
+                ListArray::from_iter_primitive::<Int32Type, _, _>(std::iter::once(Some(vec![
+                    Some(77),
+                    Some(88),
+                    Some(99),
+                ])));
+            let dataset = commit_overlay(
+                dataset,
+                "listov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(overlay_tags) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let tags = batch
+                .column(batch.schema().index_of("tags").unwrap())
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap();
+            let row1 = tags.value(0);
+            let row1 = row1.as_any().downcast_ref::<Int32Array>().unwrap();
+            let row2 = tags.value(1);
+            let row2 = row2.as_any().downcast_ref::<Int32Array>().unwrap();
+            // Offset 1 falls through to base [1, 10]; offset 2 takes the overlay.
+            assert_eq!(row1.values(), &[1, 10]);
+            assert_eq!(row2.values(), &[77, 88, 99]);
+        }
+
+        /// A top-level Map column resolves as a single atomic field even though its
+        /// value spans two leaves (key and value): both leaf ids map back to the one
+        /// Map atomic field, and the whole map value at a covered offset is replaced.
+        /// Maps require
+        /// the 2.2+ file format, so this runs only at V2_2 (unlike the V2_0/V2_1
+        /// parametrized tests).
+        #[tokio::test]
+        async fn test_map_overlay_end_to_end() {
+            use arrow_array::MapArray;
+            use arrow_array::builder::{Int32Builder, MapBuilder};
+
+            let version = LanceFileVersion::V2_2;
+
+            // Base row i holds the single entry {i: i * 10}.
+            let mut builder = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new());
+            for i in 0..6i32 {
+                builder.keys().append_value(i);
+                builder.values().append_value(i * 10);
+                builder.append(true).unwrap();
+            }
+            let base_attrs = builder.finish();
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("attrs", base_attrs.data_type().clone(), true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(base_attrs),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `attrs` (top-level field id 1) at offset 2 with a two-entry map.
+            let mut ov = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new());
+            ov.keys().append_value(7);
+            ov.values().append_value(77);
+            ov.keys().append_value(8);
+            ov.values().append_value(88);
+            ov.append(true).unwrap();
+            let overlay_attrs = ov.finish();
+            let dataset = commit_overlay(
+                dataset,
+                "mapov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(overlay_attrs) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let attrs = batch
+                .column(batch.schema().index_of("attrs").unwrap())
+                .as_any()
+                .downcast_ref::<MapArray>()
+                .unwrap();
+
+            let entries = |i: usize| -> (Vec<i32>, Vec<i32>) {
+                let row = attrs.value(i);
+                let keys = row.column(0).as_any().downcast_ref::<Int32Array>().unwrap();
+                let vals = row.column(1).as_any().downcast_ref::<Int32Array>().unwrap();
+                (keys.values().to_vec(), vals.values().to_vec())
+            };
+            // Offset 1 falls through to the base entry {1: 10}; offset 2 takes the
+            // overlay map {7: 77, 8: 88}.
+            assert_eq!(entries(0), (vec![1], vec![10]));
+            assert_eq!(entries(1), (vec![7, 8], vec![77, 88]));
+        }
+
+        /// Base `id` + a struct `s { a, b }` (6 rows). Field ids: s=1, a=2, b=3.
+        async fn create_struct_dataset(version: LanceFileVersion) -> (Dataset, Fields) {
+            let s_fields = Fields::from(vec![
+                ArrowField::new("a", DataType::Int32, true),
+                ArrowField::new("b", DataType::Int32, true),
+            ]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("s", DataType::Struct(s_fields.clone()), true),
+            ]));
+            let s = Arc::new(StructArray::new(
+                s_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), s],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+            (dataset, s_fields)
+        }
+
+        fn struct_col<'a>(batch: &'a RecordBatch, name: &str) -> &'a StructArray {
+            batch
+                .column(batch.schema().index_of(name).unwrap())
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap()
+        }
+
+        fn i32_child(s: &StructArray, i: usize) -> Int32Array {
+            s.column(i)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .clone()
+        }
+
+        /// The reviewer's core case (r3553495147): an overlay stores only sub-field
+        /// `s.a`, but the read projects the whole struct `s`. The overlay must splice
+        /// into `a` and leave `b` untouched (previously this panicked because the merge
+        /// fetched the whole `s` from an overlay file holding only `a`).
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_subfield_projecting_parent_struct(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            // Overlay ONLY `s.a` (field id 2) at offset 2.
+            let a_only = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let overlay = Arc::new(StructArray::new(
+                a_only,
+                vec![Arc::new(Int32Array::from(vec![777]))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "aov",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let s = struct_col(&batch, "s");
+            // a: offset 1 base (1), offset 2 overlaid (777).
+            assert_eq!(i32_child(s, 0).values(), &[1, 777]);
+            // b: untouched base (100, 200).
+            assert_eq!(i32_child(s, 1).values(), &[100, 200]);
+        }
+
+        /// An overlay on a non-projected sibling leaf must be skipped and its file
+        /// never opened: overlay covers `s.b`, but the read projects only `s.a`.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_nonprojected_sibling_skipped(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            let b_only = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]);
+            let overlay = Arc::new(StructArray::new(
+                b_only,
+                vec![Arc::new(Int32Array::from(vec![888]))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "bov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+            // Delete the overlay file: if projecting only `s.a` opened it, this fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/bov.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let a_only = dataset.schema().project_by_ids(&[2], true);
+            let batch = frag.take(&[1, 2], &a_only).await.unwrap();
+            let s = struct_col(&batch, "s");
+            // Only `a` is projected, unchanged base values.
+            assert_eq!(i32_child(s, 0).values(), &[1, 2]);
+        }
+
+        /// Two overlays target different sub-fields of the same struct, and a third
+        /// re-overlays `s.a`. Each leaf resolves independently and newest wins on `a`.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_multiple_subfields_newest_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            let a_field = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let b_field = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]);
+            // Older: a := 700 at offset 2.
+            let dataset = commit_overlay(
+                dataset,
+                "a_old",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    a_field.clone(),
+                    vec![Arc::new(Int32Array::from(vec![700]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+            // b := 800 at offset 2.
+            let dataset = commit_overlay(
+                dataset,
+                "b_ov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    b_field,
+                    vec![Arc::new(Int32Array::from(vec![800]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+            // Newest: a := 999 at offset 2 (shadows the older `a` overlay).
+            let dataset = commit_overlay(
+                dataset,
+                "a_new",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    a_field,
+                    vec![Arc::new(Int32Array::from(vec![999]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2], &full_schema(&dataset)).await.unwrap();
+            let s = struct_col(&batch, "s");
+            assert_eq!(i32_child(s, 0).values(), &[999]); // newest `a` wins
+            assert_eq!(i32_child(s, 1).values(), &[800]); // `b` from its own overlay
+        }
+
+        /// Three levels of nesting: `outer { middle { a, b } }`. An overlay on the
+        /// deep leaf `outer.middle.a` splices correctly when the whole `outer` is read.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_deeply_nested_subfield(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let mid_fields = Fields::from(vec![
+                ArrowField::new("a", DataType::Int32, true),
+                ArrowField::new("b", DataType::Int32, true),
+            ]);
+            let outer_fields = Fields::from(vec![ArrowField::new(
+                "middle",
+                DataType::Struct(mid_fields.clone()),
+                true,
+            )]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("outer", DataType::Struct(outer_fields.clone()), true),
+            ]));
+            // Field ids: outer=1, middle=2, a=3, b=4.
+            let middle = Arc::new(StructArray::new(
+                mid_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let outer = Arc::new(StructArray::new(outer_fields, vec![middle], None));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), outer],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay the deep leaf `outer.middle.a` (field id 3) at offset 2.
+            let a_leaf = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let mid_a = Fields::from(vec![ArrowField::new(
+                "middle",
+                DataType::Struct(a_leaf.clone()),
+                true,
+            )]);
+            let overlay = Arc::new(StructArray::new(
+                mid_a,
+                vec![Arc::new(StructArray::new(
+                    a_leaf,
+                    vec![Arc::new(Int32Array::from(vec![777]))],
+                    None,
+                ))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "deepov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let outer = struct_col(&batch, "outer");
+            let middle = outer
+                .column(0)
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            // a: offset 1 base (1), offset 2 overlaid (777); b untouched.
+            assert_eq!(i32_child(middle, 0).values(), &[1, 777]);
+            assert_eq!(i32_child(middle, 1).values(), &[100, 200]);
+
+            // Projecting the *intermediate* struct `outer.middle` (field id 2) while
+            // the overlay targets a deeper field (id 3) must still apply: the
+            // overlay's leaf id falls inside the projected subtree, so it maps to a
+            // projected atomic field. (This is the case wjones127/westonpace flagged where a
+            // top-level-only mapping would miss the overlay.)
+            let middle_only = dataset.schema().project_by_ids(&[2], true);
+            let batch = frag.take(&[2], &middle_only).await.unwrap();
+            let middle = struct_col(&batch, "outer")
+                .column(0)
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            assert_eq!(i32_child(middle, 0).values(), &[777]);
+        }
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_fragment_scan(
@@ -3182,7 +4873,7 @@ mod tests {
             updated_fragments: vec![u1.fragment],
             new_fragments: vec![],
             fields_modified: u1.fields_modified,
-            merged_generations: Vec::new(),
+            compacted_sstables: Vec::new(),
             fields_for_preserving_frag_bitmap: vec![],
             update_mode: Some(UpdateMode::RewriteColumns),
             inserted_rows_filter: None,
@@ -3263,7 +4954,7 @@ mod tests {
             updated_fragments: vec![u2.fragment],
             new_fragments: vec![],
             fields_modified: u2.fields_modified,
-            merged_generations: Vec::new(),
+            compacted_sstables: Vec::new(),
             fields_for_preserving_frag_bitmap: vec![],
             update_mode: Some(UpdateMode::RewriteColumns),
             inserted_rows_filter: None,
@@ -4359,6 +6050,54 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_indexed_metadata_heuristic_counts_selected_physical_columns() {
+        let schema = Schema::try_from(&ArrowSchema::new(vec![
+            ArrowField::new(
+                "s",
+                DataType::Struct(
+                    vec![
+                        ArrowField::new("x", DataType::Int32, true),
+                        ArrowField::new("y", DataType::Int32, true),
+                    ]
+                    .into(),
+                ),
+                true,
+            ),
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+            ArrowField::new("c", DataType::Int32, true),
+        ]))
+        .unwrap();
+        let data_file = DataFile {
+            path: "wide.lance".to_string(),
+            fields: Arc::from([0, 1, 2, 3, 4, 5]),
+            column_indices: Arc::from([-1, 0, 1, 2, 3, 4]),
+            file_major_version: 2,
+            file_minor_version: 1,
+            file_size_bytes: CachedFileSize::unknown(),
+            base_id: None,
+        };
+
+        let full_struct =
+            ReaderProjection::from_column_names(LanceFileVersion::V2_1, &schema, &["s"]).unwrap();
+        assert_eq!(full_struct.column_indices.len(), 2);
+        assert!(!FileFragment::should_try_indexed_metadata(
+            &data_file,
+            &full_struct,
+            LanceFileVersion::V2_1
+        ));
+
+        let partial_struct =
+            ReaderProjection::from_column_names(LanceFileVersion::V2_1, &schema, &["s.x"]).unwrap();
+        assert_eq!(partial_struct.column_indices.len(), 1);
+        assert!(FileFragment::should_try_indexed_metadata(
+            &data_file,
+            &partial_struct,
+            LanceFileVersion::V2_1
+        ));
+    }
+
     #[tokio::test]
     async fn test_iops_read_small() {
         // Create a file that has 8 columns.
@@ -4420,4 +6159,80 @@ mod tests {
         assert_io_eq!(stats, read_iops, 1);
         assert_io_lt!(stats, read_bytes, 4096);
     }
+
+    #[tokio::test]
+    async fn test_update_columns_with_json_extension_type() {
+        use arrow_array::UInt64Array;
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::ARROW_JSON_EXT_NAME;
+        use lance_core::ROW_ID;
+        use std::collections::HashMap;
+
+        // Create a dataset with an Arrow JSON extension column
+        let test_dir = TempStrDir::default();
+        let mut json_metadata = HashMap::new();
+        json_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            ARROW_JSON_EXT_NAME.to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int64, false),
+            ArrowField::new("name", DataType::Utf8, true),
+            ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])),
+                Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"x":1}"#,
+                    r#"{"x":2}"#,
+                    r#"{"x":3}"#,
+                    r#"{"x":4}"#,
+                    r#"{"x":5}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let dataset = Dataset::write(reader, test_dir.as_ref(), None)
+            .await
+            .unwrap();
+
+        // Build the right stream with Arrow JSON column (Utf8 + arrow.json extension)
+        // Only update rows with row_id 1 and 3
+        let update_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new(ROW_ID, DataType::UInt64, false),
+            ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata),
+        ]));
+        let update_batch = RecordBatch::try_new(
+            update_schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![1, 3])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true,"id":2}"#,
+                    r#"{"updated":true,"id":4}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let right_stream: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(update_batch)],
+            update_schema,
+        ));
+
+        // Perform update_columns - this should NOT fail with type mismatch
+        // Previously this would error with:
+        //   "It is not possible to interleave arrays of different data types (Utf8 and LargeBinary)"
+        let mut fragment = dataset.get_fragment(0).unwrap();
+        let (updated_fragment, fields_modified) = fragment
+            .update_columns(right_stream, ROW_ID, ROW_ID)
+            .await
+            .unwrap();
+
+        // Verify the operation produced valid results
+        assert!(!fields_modified.is_empty());
+        assert!(!updated_fragment.files.is_empty());
+    }
 }
diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs
index a61f0e0c46a..2ab3bd1d92b 100644
--- a/rust/lance/src/dataset/fragment/write.rs
+++ b/rust/lance/src/dataset/fragment/write.rs
@@ -21,6 +21,7 @@ use uuid::Uuid;
 
 use crate::Result;
 use crate::dataset::builder::DatasetBuilder;
+use crate::dataset::utils::SchemaAdapter;
 use crate::dataset::write::{do_write_fragments, validate_and_resolve_target_bases_with_primary};
 use crate::dataset::{DATA_DIR, Dataset, ReadParams, WriteMode, WriteParams};
 
@@ -106,6 +107,12 @@ impl<'a> FragmentCreateBuilder<'a> {
         id: Option<u64>,
     ) -> Result<Fragment> {
         let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?;
+        // Convert Arrow JSON columns (`arrow.json`, stored as Utf8) into Lance JSON
+        // (`lance.json`, stored as JSONB-encoded LargeBinary) before writing. The
+        // multi-fragment and dataset write paths perform this through `do_write_fragments`;
+        // the single-fragment create path must do the same or the raw UTF-8 string bytes
+        // would be written into a column whose schema declares JSONB, corrupting reads.
+        let stream = SchemaAdapter::new(stream.schema()).to_physical_stream(stream);
         self.write_impl(stream, schema, id).await
     }
 
@@ -243,6 +250,7 @@ impl<'a> FragmentCreateBuilder<'a> {
             params,
             version,
             target_bases_info,
+            Vec::new(),
         )
         .await
     }
@@ -415,7 +423,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Cannot write with an empty schema.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing empty reader produces an error
@@ -429,7 +437,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Input data was empty.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing with incorrect schema produces an error.
@@ -447,7 +455,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
             if difference.contains("fields did not match")),
             "{:?}",
-            &result
+            result
         );
     }
 
@@ -506,7 +514,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Cannot write with an empty schema.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing empty reader produces an error
@@ -533,7 +541,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
             if difference.contains("fields did not match")),
             "{:?}",
-            &result
+            result
         );
     }
 
diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs
index 7cac7815125..c8f23e9720d 100644
--- a/rust/lance/src/dataset/index.rs
+++ b/rust/lance/src/dataset/index.rs
@@ -17,7 +17,7 @@ use arrow_schema::DataType;
 use async_trait::async_trait;
 use lance_core::{Error, Result};
 use lance_encoding::version::LanceFileVersion;
-use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
+use lance_index::is_system_index;
 use lance_index::pb::VectorIndexDetails;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_table::format::IndexMetadata;
@@ -29,19 +29,39 @@ use super::optimize::{IndexRemapper, IndexRemapperOptions};
 #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DatasetIndexRemapperOptions {}
 
+/// Loads index metadata when compaction has at least one index to remap.
+///
+/// Returns all index metadata, including system indices, so the remapper uses a
+/// consistent snapshot. Returns `None` when there are no non-system indices.
+pub(crate) async fn load_indices_for_remapping(
+    dataset: &Dataset,
+) -> Result<Option<Arc<Vec<IndexMetadata>>>> {
+    if dataset.manifest.index_section.is_none() {
+        return Ok(None);
+    }
+
+    let indices = dataset.load_indices().await?;
+    let has_remappable_index = indices.iter().any(|index| !is_system_index(index));
+    Ok(has_remappable_index.then_some(indices))
+}
+
+#[async_trait]
 impl IndexRemapperOptions for DatasetIndexRemapperOptions {
-    fn create_remapper(
-        &self,
-        dataset: &Dataset,
-    ) -> crate::Result<Box<dyn super::optimize::IndexRemapper>> {
-        Ok(Box::new(DatasetIndexRemapper {
+    async fn create_remapper(&self, dataset: &Dataset) -> Result<Option<Box<dyn IndexRemapper>>> {
+        let Some(indices) = load_indices_for_remapping(dataset).await? else {
+            return Ok(None);
+        };
+
+        Ok(Some(Box::new(DatasetIndexRemapper {
             dataset: Arc::new(dataset.clone()),
-        }))
+            indices,
+        })))
     }
 }
 
 struct DatasetIndexRemapper {
     dataset: Arc<Dataset>,
+    indices: Arc<Vec<IndexMetadata>>,
 }
 
 impl DatasetIndexRemapper {
@@ -62,10 +82,9 @@ impl IndexRemapper for DatasetIndexRemapper {
         affected_fragment_ids: &[u64],
     ) -> Result<Vec<RemappedIndex>> {
         let affected_frag_ids = HashSet::<u64>::from_iter(affected_fragment_ids.iter().copied());
-        let indices = self.dataset.load_indices().await?;
-        let mut remapped = Vec::with_capacity(indices.len());
-        for index in indices.iter() {
-            let needs_remapped = index.name != FRAG_REUSE_INDEX_NAME
+        let mut remapped = Vec::with_capacity(self.indices.len());
+        for index in self.indices.iter() {
+            let needs_remapped = !is_system_index(index)
                 && match &index.fragment_bitmap {
                     None => true,
                     Some(fragment_bitmap) => fragment_bitmap
@@ -73,7 +92,12 @@ impl IndexRemapper for DatasetIndexRemapper {
                         .any(|frag_idx| affected_frag_ids.contains(&(frag_idx as u64))),
                 };
             if needs_remapped {
-                let remap_result = self.remap_index(index, &mapping).await?;
+                // Box the remap future at the call site: inlining `remap_index` into this
+                // loop's async layout otherwise exceeds rustc's depth limit. It has to be
+                // boxed here, not inside `remap_index` — boxing internally turns the
+                // future's `Send` check into a `Box<Future>: Send` trait obligation that
+                // overflows the solver through the cache types (E0275 downstream).
+                let remap_result = Box::pin(self.remap_index(index, &mapping)).await?;
                 match remap_result {
                     RemapResult::Drop => continue,
                     RemapResult::Keep(id) => {
@@ -178,13 +202,55 @@ impl LanceIndexStoreExt for LanceIndexStore {
 mod tests {
     use super::*;
     use crate::dataset::WriteParams;
+    use crate::dataset::transaction::{Operation, Transaction};
     use crate::index::DatasetIndexExt;
+    use crate::index::frag_reuse::build_frag_reuse_index_metadata;
     use crate::index::vector::VectorIndexParams;
     use lance_datagen::{BatchCount, RowCount, array};
     use lance_index::IndexType;
+    use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndexDetails};
     use lance_linalg::distance::MetricType;
     use uuid::Uuid;
 
+    #[tokio::test]
+    async fn test_remapper_not_created_without_remappable_indices() {
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .into_reader_rows(RowCount::from(1), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap();
+        let options = DatasetIndexRemapperOptions::default();
+
+        assert!(options.create_remapper(&dataset).await.unwrap().is_none());
+
+        let frag_reuse_index = build_frag_reuse_index_metadata(
+            &dataset,
+            None,
+            FragReuseIndexDetails {
+                versions: Vec::new(),
+            },
+            Default::default(),
+        )
+        .await
+        .unwrap();
+        let transaction = Transaction::new(
+            dataset.manifest.version,
+            Operation::CreateIndex {
+                new_indices: vec![frag_reuse_index],
+                removed_indices: Vec::new(),
+            },
+            None,
+        );
+        dataset
+            .apply_commit(transaction, &Default::default(), &Default::default())
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].name, FRAG_REUSE_INDEX_NAME);
+        assert!(options.create_remapper(&dataset).await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn test_remapper_only_touches_segments_with_affected_fragments() {
         let test_dir = tempfile::tempdir().unwrap();
@@ -264,12 +330,14 @@ mod tests {
                         .as_ref()
                         .expect("test segment metadata should have fragment coverage")
                         .iter(),
+                    segment.fields.iter().copied(),
                     segment
                         .index_details
                         .as_ref()
                         .expect("test segment metadata should have index details")
                         .clone(),
                     segment.index_version,
+                    segment.dataset_version,
                 )
             })
             .collect::<Vec<_>>();
@@ -296,7 +364,9 @@ mod tests {
 
         let remapper = DatasetIndexRemapperOptions::default()
             .create_remapper(&dataset)
-            .unwrap();
+            .await
+            .unwrap()
+            .expect("vector index should require a remapper");
         let remapped = remapper
             .remap_indices(RowAddrRemap::empty(), &[target_fragments[0].id() as u64])
             .await
diff --git a/rust/lance/src/dataset/index/frag_reuse.rs b/rust/lance/src/dataset/index/frag_reuse.rs
index ceebe456bbf..08a3e8f42fd 100644
--- a/rust/lance/src/dataset/index/frag_reuse.rs
+++ b/rust/lance/src/dataset/index/frag_reuse.rs
@@ -16,15 +16,34 @@ use roaring::RoaringBitmap;
 /// If all the indices currently available are already caught up to as a specific reuse version,
 /// all older reuse versions (inclusive) can be cleaned up.
 ///
-/// An index is considered caught up against a specific reuse version if
-/// 1. the index is created after or at the same dataset version as the reuse version
-/// 2. there is no old fragment in the version that is covered by the index and can be remapped.
-///    If an index's fragment bitmap is missing, we will consider it as caught up.
-///    Otherwise, we will never be able to clean up the reuse version.
+/// An index is considered caught up against a specific reuse version if either:
+/// 1. its coverage is disjoint from the fragments the reuse chain touches, so it
+///    holds nothing the FRI would remap (the common multi-index case: a
+///    compaction rewrote a sibling index's fragments, not this one); or
+/// 2. it is at or past the reuse version's dataset version and no old fragment
+///    in the version is still in its bitmap. A missing bitmap counts as caught
+///    up, else the version could never be cleaned up.
 ///
 /// Note that there could be a race condition that an index is being added during the cleanup,
 /// This will make that specific index not efficient until the next reindex,
 /// but it will not cause any correctness problem.
+///
+/// Typically run after [`compact_files`] with deferred remap and per-index
+/// [`remap_column_index`] have caught the indexes up.
+///
+/// # Example
+///
+/// ```no_run
+/// # use lance::dataset::index::frag_reuse::cleanup_frag_reuse_index;
+/// # async fn example(dataset: &mut lance::Dataset) -> lance::Result<()> {
+/// // Trim the fragment-reuse index to the versions still needed by some index.
+/// cleanup_frag_reuse_index(dataset).await?;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`compact_files`]: crate::dataset::optimize::compact_files
+/// [`remap_column_index`]: crate::dataset::optimize::remapping::remap_column_index
 pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Result<()> {
     // check against index metadata before auto-remap
     let indices = read_manifest_indexes(
@@ -42,12 +61,14 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu
         .await
         .unwrap();
 
+    let chain_frag_bitmap = reuse_chain_frag_bitmap(&frag_reuse_details.versions);
+
     let mut retained_versions = Vec::new();
     let mut fragment_bitmaps = RoaringBitmap::new();
     for version in frag_reuse_details.versions.iter() {
         let check_results = indices
             .iter()
-            .map(|idx| is_index_remap_caught_up(version, idx))
+            .map(|idx| is_index_remap_caught_up(version, idx, &chain_frag_bitmap))
             .collect::<Vec<_>>();
 
         if check_results
@@ -97,14 +118,38 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu
     Ok(())
 }
 
+/// Every fragment the reuse chain touches (old + new) across all versions. An
+/// index disjoint from this set holds no row address the FRI remaps, so trimming
+/// can never strand it (fragment ids are never reused).
+fn reuse_chain_frag_bitmap(versions: &[FragReuseVersion]) -> RoaringBitmap {
+    let mut bitmap = RoaringBitmap::new();
+    for version in versions {
+        bitmap.extend(version.old_frag_ids().iter().map(|&id| id as u32));
+        bitmap.extend(version.new_frag_ids().iter().map(|&id| id as u32));
+    }
+    bitmap
+}
+
 fn is_index_remap_caught_up(
     frag_reuse_version: &FragReuseVersion,
     index_meta: &IndexMetadata,
+    chain_frag_bitmap: &RoaringBitmap,
 ) -> lance_core::Result<bool> {
     if is_system_index(index_meta) {
         return Ok(true);
     }
 
+    // Disjoint coverage => caught up regardless of dataset_version, bypassing the
+    // stale-version gate below (see fn docs). The chain includes NEW fragments
+    // deliberately: a deferred-remap commit advances a covering index's bitmap
+    // onto them before its data is remapped, so an old-frag-only check would
+    // clear a still-stale index and trim a version it needs.
+    if let Some(index_frag_bitmap) = &index_meta.fragment_bitmap
+        && index_frag_bitmap.is_disjoint(chain_frag_bitmap)
+    {
+        return Ok(true);
+    }
+
     if index_meta.dataset_version < frag_reuse_version.dataset_version {
         return Ok(false);
     }
@@ -157,6 +202,105 @@ mod tests {
     use lance_index::IndexType;
     use lance_index::scalar::ScalarIndexParams;
 
+    fn frag_digest(id: u64) -> lance_index::frag_reuse::FragDigest {
+        lance_index::frag_reuse::FragDigest {
+            id,
+            physical_rows: 100,
+            num_deleted_rows: 0,
+        }
+    }
+
+    fn reuse_version(dataset_version: u64, old: &[u64], new: &[u64]) -> FragReuseVersion {
+        FragReuseVersion {
+            dataset_version,
+            groups: vec![lance_index::frag_reuse::FragReuseGroup {
+                changed_row_addrs: Vec::new(),
+                old_frags: old.iter().copied().map(frag_digest).collect(),
+                new_frags: new.iter().copied().map(frag_digest).collect(),
+            }],
+        }
+    }
+
+    fn index_covering(dataset_version: u64, covered: &[u32]) -> IndexMetadata {
+        IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            fields: vec![0],
+            name: "test_idx".into(),
+            dataset_version,
+            fragment_bitmap: Some(RoaringBitmap::from_iter(covered.iter().copied())),
+            index_details: None,
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        }
+    }
+
+    /// The catch-up determination must not pin the FRI on an index that is
+    /// simply unrelated to the compaction, while still retaining versions that a
+    /// covering-but-not-yet-remapped index needs.
+    #[test]
+    fn test_caught_up_uses_fragment_coverage_not_only_version() {
+        // A reuse version at dataset_version 10 rewrote fragments [4, 5] -> [6].
+        let version = reuse_version(10, &[4, 5], &[6]);
+        let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version));
+
+        // Non-covering, stale version: touches none of the rewritten frags, so
+        // caught up despite version 5 < 10 (the case the old gate got wrong).
+        assert_true!(
+            is_index_remap_caught_up(&version, &index_covering(5, &[1, 2, 3]), &chain).unwrap()
+        );
+
+        // Still holds an old fragment: not caught up.
+        assert_false!(
+            is_index_remap_caught_up(&version, &index_covering(5, &[1, 4, 5]), &chain).unwrap()
+        );
+
+        // Bitmap advanced onto the new fragment but data not yet remapped: not
+        // caught up (why the chain must include new frags).
+        assert_false!(
+            is_index_remap_caught_up(&version, &index_covering(5, &[1, 6]), &chain).unwrap()
+        );
+
+        // Once remapped (version advanced): caught up.
+        assert_true!(
+            is_index_remap_caught_up(&version, &index_covering(11, &[1, 6]), &chain).unwrap()
+        );
+    }
+
+    /// The chain spans every reuse version, not just the one being checked: a
+    /// stale index touching only a *later* version's fragment must still fall to
+    /// the version gate (a per-version chain would wrongly clear it).
+    #[test]
+    fn test_caught_up_uses_whole_reuse_chain() {
+        let v1 = reuse_version(10, &[4, 5], &[6]); // 4,5 -> 6
+        let v2 = reuse_version(11, &[6], &[7]); // 6 -> 7
+        let chain = reuse_chain_frag_bitmap(&[v1.clone(), v2]);
+
+        // Stale index (version 5) covering only v2's new fragment [7]: not
+        // disjoint from the chain, so not caught up on v1.
+        assert_false!(is_index_remap_caught_up(&v1, &index_covering(5, &[1, 7]), &chain).unwrap());
+    }
+
+    /// Whole-fragment removal (every row deleted, no replacement): an index
+    /// emptied by the deletion has an empty bitmap and must count as caught up --
+    /// it holds only dead rows -- else its stale version pins the removed-fragment
+    /// version forever (remap hits the drop-everything path, never advancing it).
+    #[test]
+    fn test_caught_up_handles_fragment_removal() {
+        // Reuse version 20 removed fragment [7] outright (no replacement).
+        let version = reuse_version(20, &[7], &[]);
+        let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version));
+
+        // Index emptied by the deletion (empty bitmap): caught up.
+        assert_true!(is_index_remap_caught_up(&version, &index_covering(5, &[]), &chain).unwrap());
+
+        // Bitmap still lists the removed fragment (not yet updated): retained.
+        assert_false!(
+            is_index_remap_caught_up(&version, &index_covering(5, &[7]), &chain).unwrap()
+        );
+    }
+
     #[tokio::test]
     async fn test_cleanup_frag_reuse_index() {
         let mut dataset = lance_datagen::gen_batch()
@@ -209,7 +353,12 @@ mod tests {
         let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap();
         // Should not be considered caught up because index was created at an old dataset version
         assert_false!(
-            is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap()
+            is_index_remap_caught_up(
+                &frag_reuse_details.versions[0],
+                scalar_index,
+                &reuse_chain_frag_bitmap(&frag_reuse_details.versions),
+            )
+            .unwrap()
         );
 
         // Remap and check index is caught up
@@ -219,7 +368,12 @@ mod tests {
         let indices = dataset.load_indices().await.unwrap();
         let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap();
         assert_true!(
-            is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap()
+            is_index_remap_caught_up(
+                &frag_reuse_details.versions[0],
+                scalar_index,
+                &reuse_chain_frag_bitmap(&frag_reuse_details.versions),
+            )
+            .unwrap()
         );
 
         // Cleanup frag reuse index and check there is no reuse version
@@ -313,7 +467,12 @@ mod tests {
                 .find(|idx| idx.name == format!("{col}_idx"))
                 .unwrap();
             assert!(
-                is_index_remap_caught_up(&frag_reuse_details.versions[0], index).unwrap(),
+                is_index_remap_caught_up(
+                    &frag_reuse_details.versions[0],
+                    index,
+                    &reuse_chain_frag_bitmap(&frag_reuse_details.versions),
+                )
+                .unwrap(),
                 "index {col}_idx was not caught up after remap"
             );
         }
diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs
index f5b89d06ff4..784ee1fa76c 100644
--- a/rust/lance/src/dataset/mem_wal.rs
+++ b/rust/lance/src/dataset/mem_wal.rs
@@ -52,7 +52,7 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 /// Column name for the mem_wal tombstone (delete sentinel) marker.
 ///
 /// `_tombstone` is a *physical* column present only in mem_wal memtables and
-/// flushed generations — it is deliberately kept out of the base table (hard
+/// SSTables — it is deliberately kept out of the base table (hard
 /// delete), so it is **not** a virtual [`is_system_column`](lance_core::is_system_column).
 /// A row with `_tombstone = true` is a delete sentinel: the newest value for
 /// its primary key, carrying null in every non-PK column, that wins
diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs
index 597c65dce83..005b47d2b53 100644
--- a/rust/lance/src/dataset/mem_wal/api.rs
+++ b/rust/lance/src/dataset/mem_wal/api.rs
@@ -14,7 +14,6 @@ use async_trait::async_trait;
 use lance_core::{Error, Result};
 use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, ShardingField, ShardingSpec};
 use lance_index::vector::hnsw::builder::HnswBuildParams;
-use lance_io::object_store::ObjectStore;
 use uuid::Uuid;
 
 use crate::Dataset;
@@ -25,8 +24,9 @@ use crate::index::DatasetIndexInternalExt;
 use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta};
 
 use super::ShardWriterConfig;
-use super::scanner::flushed_cache::open_flushed_dataset;
+use super::scanner::sstable_cache::open_sstable;
 use super::scanner::{DatasetCache, ShardSnapshot};
+use super::util::derived_store_params;
 use super::write::MemIndexConfig;
 use super::write::ShardWriter;
 
@@ -392,10 +392,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap<String, Stri
             "durable_write".to_string(),
             config.durable_write.to_string(),
         ),
-        (
-            "sync_indexed_write".to_string(),
-            config.sync_indexed_write.to_string(),
-        ),
         (
             "max_wal_buffer_size".to_string(),
             config.max_wal_buffer_size.to_string(),
@@ -432,14 +428,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap<String, Stri
             "backpressure_log_interval_ms".to_string(),
             config.backpressure_log_interval.as_millis().to_string(),
         ),
-        (
-            "async_index_buffer_rows".to_string(),
-            config.async_index_buffer_rows.to_string(),
-        ),
-        (
-            "async_index_interval_ms".to_string(),
-            config.async_index_interval.as_millis().to_string(),
-        ),
         (
             "enable_memtable".to_string(),
             config.enable_memtable.to_string(),
@@ -491,10 +479,10 @@ pub trait DatasetMemWalExt {
         Ok(Vec::new())
     }
 
-    /// Prewarm the flushed generations of the given MemWAL shards into this
+    /// Prewarm the SSTables of the given MemWAL shards into this
     /// dataset's session caches.
     ///
-    /// For every flushed generation in `snapshots`, opens the generation's
+    /// For every SSTable in `snapshots`, opens the generation's
     /// on-disk dataset (populating the session's metadata/index caches, and the
     /// optional `cache` of opened `Arc<Dataset>`s) and prewarms each of its
     /// indexes. Opens run concurrently.
@@ -597,7 +585,9 @@ impl DatasetMemWalExt for Dataset {
         cache: Option<&Arc<dyn DatasetCache>>,
     ) -> Result<()> {
         let session = self.session();
-        // Resolve flushed paths exactly as the LSM collector does, so the
+        // Every open below targets a generation URI, never the base's own.
+        let store_params = self.store_params().map(derived_store_params);
+        // Resolve SSTable paths exactly as the LSM collector does, so the
         // session/cache entries we warm key-match the paths later lookups open.
         let base_path = self.uri().trim_end_matches('/').to_string();
         let opens = snapshots
@@ -606,11 +596,13 @@ impl DatasetMemWalExt for Dataset {
                 let shard_id = snapshot.shard_id;
                 let base_path = &base_path;
                 let session = &session;
-                snapshot.flushed_generations.iter().map(move |flushed| {
-                    let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, flushed.path);
+                let store_params = &store_params;
+                snapshot.sstables.iter().map(move |sstable| {
+                    let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, sstable.path);
                     async move {
                         let dataset =
-                            open_flushed_dataset(&path, Some(session), cache, None).await?;
+                            open_sstable(&path, Some(session), store_params.as_ref(), cache, None)
+                                .await?;
                         prewarm_all_indexes(&dataset).await
                     }
                 })
@@ -700,9 +692,17 @@ impl DatasetMemWalExt for Dataset {
         // Set shard_id in config
         config.shard_id = shard_id;
 
-        // Get object store and base path
+        // Inject the dataset's store params + session so the flusher opens the
+        // base + generations with the same store the base was resolved with.
+        config.store_params = self.store_params().cloned();
+        config.session = Some(self.session());
+
+        // Reuse the dataset's own object store + base path; `ObjectStore::from_uri`
+        // would discard the store params the dataset was opened with, signing WAL
+        // writes with the ambient identity. Mirrors `list_mem_wal_latest_shard_ids`.
         let base_uri = self.uri();
-        let (store, base_path) = ObjectStore::from_uri(base_uri).await?;
+        let store = self.object_store(None).await?;
+        let base_path = self.branch_location().path;
 
         // Create ShardWriter
         ShardWriter::open(
@@ -771,7 +771,7 @@ async fn load_vector_index_config(
 
 #[cfg(test)]
 mod tests {
-    use super::super::scanner::FlushedMemTableCache;
+    use super::super::scanner::SsTableCache;
     use super::*;
 
     use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
@@ -802,9 +802,9 @@ mod tests {
 
     #[tokio::test]
     async fn test_prewarm_mem_wal_opens_and_warms_indexes() {
-        // `prewarm_mem_wal` opens each flushed generation (into the base
+        // `prewarm_mem_wal` opens each SSTable (into the base
         // dataset's session + the supplied cache) and warms its indexes. We
-        // place a flushed-generation dataset with a BTree index at the
+        // place an SSTable dataset with a BTree index at the
         // canonical `{base}/_mem_wal/{shard}/{folder}` path, prewarm it via a
         // snapshot, and assert the generation is cached and its index loadable.
         let tmp = tempfile::tempdir().unwrap();
@@ -817,7 +817,7 @@ mod tests {
             .await
             .unwrap();
 
-        // Flushed generation with a BTree index on `id`.
+        // SSTable with a BTree index on `id`.
         let shard_id = Uuid::new_v4();
         let folder = "deadbeef_gen_1";
         let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, folder);
@@ -839,9 +839,9 @@ mod tests {
 
         let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, folder.to_string());
+            .with_sstable(1, folder.to_string());
 
-        let cache: Arc<dyn DatasetCache> = Arc::new(FlushedMemTableCache::new(4));
+        let cache: Arc<dyn DatasetCache> = Arc::new(SsTableCache::new(4));
         base.prewarm_mem_wal(std::slice::from_ref(&snapshot), Some(&cache))
             .await
             .expect("prewarm must open the generation and warm its index");
@@ -849,7 +849,7 @@ mod tests {
         // The generation is resident in the cache (same session), with its
         // index loadable — a later lookup that opens this path is a pure hit.
         let warmed = cache
-            .get_or_open(&gen_uri, Some(base.session()))
+            .get_or_open(&gen_uri, Some(base.session()), base.store_params().cloned())
             .await
             .unwrap();
         assert_eq!(warmed.load_indices().await.unwrap().len(), 1);
@@ -857,7 +857,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_prewarm_mem_wal_empty_is_noop() {
-        // No snapshots / no flushed generations: prewarm is a clean no-op.
+        // No snapshots / no SSTables: prewarm is a clean no-op.
         let tmp = tempfile::tempdir().unwrap();
         let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
         let schema = id_v_schema();
diff --git a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
index 4009557702c..570122581b7 100644
--- a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
+++ b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
@@ -594,9 +594,12 @@ impl HnswGraph {
     }
 
     fn validate_source(&self, vectors: &impl VectorSource, needed_len: usize) -> Result<()> {
+        // Not caller input: the graph was sized below what the memtable holds.
+        // See the matching note in `storage.rs::append_batch`.
         if needed_len > self.nodes.len() {
-            return Err(Error::invalid_input(format!(
-                "graph capacity {} exhausted: need {needed_len}",
+            return Err(Error::internal(format!(
+                "HNSW graph capacity {} exhausted: need {needed_len}; \
+                 the graph is sized below the memtable's row capacity",
                 self.nodes.len()
             )));
         }
diff --git a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
index bbeb57a5fe2..c0ee9d57ff6 100644
--- a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
+++ b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
@@ -269,25 +269,30 @@ impl ArrowFixedSizeListVectorStore {
             )));
         };
 
+        // Exhaustion is a shard-construction bug (store sized below the memtable),
+        // not caller input — hence `internal`, not `invalid_input`.
         let start = self.committed_len.load(Ordering::Relaxed);
         let end = start.checked_add(num_rows).ok_or_else(|| {
-            Error::invalid_input(format!(
+            Error::internal(format!(
                 "vector count overflow: start={}, batch_len={}",
                 start, num_rows
             ))
         })?;
         if end > self.capacity {
-            return Err(Error::invalid_input(format!(
-                "capacity {} exhausted: inserting rows [{}..{})",
+            return Err(Error::internal(format!(
+                "HNSW vector store capacity {} exhausted: inserting rows [{}..{}); \
+                 the store is sized below the memtable's row capacity",
                 self.capacity, start, end
             )));
         }
 
         let batch_idx = self.committed_batches.load(Ordering::Relaxed);
         if batch_idx >= self.max_batches {
-            return Err(Error::invalid_input(format!(
-                "max_batches {} exhausted",
-                self.max_batches
+            return Err(Error::internal(format!(
+                "HNSW vector store max_batches {} exhausted at batch_idx {} \
+                 (inserting rows [{}..{})); the store is sized below the \
+                 memtable's batch capacity",
+                self.max_batches, batch_idx, start, end
             )));
         }
 
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index d16d3105551..b3da5b5b67e 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -23,11 +23,14 @@ mod pk_key;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::time::Instant;
 
 use datafusion::common::ScalarValue;
 
 use super::memtable::batch_store::StoredBatch;
+use super::wal::WriterCursors;
 use arrow_array::RecordBatch;
+use arrow_schema::{DataType, Schema as ArrowSchema};
 use lance_core::datatypes::Schema as LanceSchema;
 use lance_core::{Error, Result};
 use lance_index::pbold;
@@ -58,6 +61,19 @@ use pk_key::encode_pk_batch;
 /// [`BTreeMemIndex`]'s byte backend indexes it directly.
 const PK_KEY_COLUMN: &str = "__pk_key__";
 
+/// Row count at or below which [`IndexStore::insert_batches`] indexes inline
+/// rather than spawning a thread per index.
+///
+/// The spawn is one OS thread *per index* — tens of microseconds each, and a table can
+/// carry several BTrees alongside its HNSW and FTS — so for a small batch it costs more
+/// than the indexing it parallelizes. Small batches are not the exceptional case: a
+/// durable put triggers a WAL flush covering only the batch it just inserted, so this
+/// path is routinely called with a single short batch.
+///
+/// The crossover depends on per-row HNSW cost, which varies with dimension and
+/// `ef_construction`; tune against `benches/mem_wal/vector/mem_wal_index_micro.rs`.
+const PARALLEL_INDEX_MIN_ROWS: usize = 64;
+
 /// The memtable's primary-key index, used to answer "newest visible version of
 /// this key" for dedup. Single-column PKs reuse the column's compact typed
 /// [`BTreeMemIndex`] (no second copy); composite PKs key a `BTreeMemIndex` on
@@ -80,6 +96,169 @@ enum PkIndex {
 // Index Store
 // ============================================================================
 
+/// Validate every configured in-memory index, and the composite primary key,
+/// against the shard schema. Call once at shard open, before any write can land.
+///
+/// This is what makes poison-and-replay *terminating*. An index insert that
+/// fails deterministically on a row that is already WAL-durable cannot be
+/// recovered from: the writer poisons, the operator reopens, replay re-reads the
+/// same WAL rows, the same insert fails again, and `open()` propagates it — a
+/// shard that never comes back. Every such failure is an index *config*
+/// disagreeing with the schema, never a property of the data, so one pass here
+/// closes the whole class before a single row is accepted.
+///
+/// The data-dependent errors inside the index layer are already unreachable
+/// through `put`: `MemTable::insert_batches_only` does a full `Arc<Schema>`
+/// equality check, so a batch that would trip one is rejected before it reaches
+/// the batch store, let alone the WAL.
+///
+/// It also rejects a config whose `field_id` names a different column than its
+/// `column`. Index *selection* keys off `field_id` — a single-column PK reuses
+/// the BTree whose `field_id` matches its key — so a config resolved only by name
+/// could be bound under the wrong identity, serving stale reads and flushing the
+/// wrong column into the durable PK sidecar. `lance_schema` supplies the
+/// authoritative name→id mapping.
+pub fn validate_index_configs(
+    configs: &[MemIndexConfig],
+    schema: &ArrowSchema,
+    lance_schema: &LanceSchema,
+    pk_columns: &[String],
+) -> Result<()> {
+    for config in configs {
+        let column = config.column();
+        let field = schema.field_with_name(column).map_err(|_| {
+            Error::invalid_input(format!(
+                "index '{}' is configured on column '{}', which is not in the shard schema; \
+                 available columns: [{}]",
+                config.name(),
+                column,
+                schema
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().as_str())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ))
+        })?;
+
+        match config {
+            // BTree falls back to per-row `ScalarValue` extraction, so it
+            // accepts any column type the schema can hold. Existence is the
+            // only precondition.
+            MemIndexConfig::BTree(_) => {}
+            MemIndexConfig::Fts(_) => {
+                if !matches!(
+                    field.data_type(),
+                    DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+                ) {
+                    return Err(Error::invalid_input(format!(
+                        "FTS index '{}' requires a Utf8, LargeUtf8, or Utf8View column; \
+                         column '{}' is {:?}",
+                        config.name(),
+                        column,
+                        field.data_type()
+                    )));
+                }
+            }
+            MemIndexConfig::Hnsw(_) => match field.data_type() {
+                DataType::FixedSizeList(item, dim) => {
+                    if item.data_type() != &DataType::Float32 {
+                        return Err(Error::invalid_input(format!(
+                            "HNSW index '{}' requires a FixedSizeList<Float32> column; \
+                             column '{}' has item type {:?}",
+                            config.name(),
+                            column,
+                            item.data_type()
+                        )));
+                    }
+                    // `HnswMemIndex.dim` is a placeholder until the first batch
+                    // pins it (`hnsw.rs`), so a zero-width vector would only
+                    // surface at insert time — i.e. on already-durable data.
+                    if *dim <= 0 {
+                        return Err(Error::invalid_input(format!(
+                            "HNSW index '{}' requires a vector dimension > 0; column '{}' has \
+                             dimension {dim}",
+                            config.name(),
+                            column,
+                        )));
+                    }
+                }
+                other => {
+                    return Err(Error::invalid_input(format!(
+                        "HNSW index '{}' requires a FixedSizeList<Float32> column; \
+                         column '{}' is {:?}",
+                        config.name(),
+                        column,
+                        other
+                    )));
+                }
+            },
+        }
+
+        // The column resolves, but index selection keys off `field_id`, not name.
+        // A config whose `field_id` identifies a *different* column would be bound
+        // under the wrong identity (e.g. reused as the single-column PK index), so
+        // reject any `field_id` that does not name the resolved column.
+        let resolved_field_id = lance_schema
+            .field(column)
+            .expect("column resolved in the Arrow schema is present in the Lance schema")
+            .id;
+        if resolved_field_id != config.field_id() {
+            return Err(Error::invalid_input(format!(
+                "index '{}' is configured with field_id {} but its column '{}' has field_id {} \
+                 in the shard schema",
+                config.name(),
+                config.field_id(),
+                column,
+                resolved_field_id,
+            )));
+        }
+    }
+
+    // Every PK column must exist in the schema. A single-column PK aliases a
+    // BTree entry (any type); only a *composite* PK builds an order-preserving
+    // encoded key, and only some types encode.
+    for column in pk_columns {
+        let field = schema.field_with_name(column).map_err(|_| {
+            Error::invalid_input(format!(
+                "primary-key column '{column}' is not in the shard schema"
+            ))
+        })?;
+        if pk_columns.len() > 1 && !is_encodable_pk_type(field.data_type()) {
+            return Err(Error::invalid_input(format!(
+                "composite primary-key column '{column}' has type {:?}, which has no \
+                 order-preserving key encoding",
+                field.data_type()
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+/// Types `pk_key::encode_value` can encode into an order-preserving composite key.
+fn is_encodable_pk_type(data_type: &DataType) -> bool {
+    matches!(
+        data_type,
+        DataType::Int8
+            | DataType::Int16
+            | DataType::Int32
+            | DataType::Int64
+            | DataType::UInt8
+            | DataType::UInt16
+            | DataType::UInt32
+            | DataType::UInt64
+            | DataType::Date32
+            | DataType::Date64
+            | DataType::Boolean
+            | DataType::Utf8
+            | DataType::LargeUtf8
+            | DataType::Binary
+            | DataType::LargeBinary
+            | DataType::FixedSizeBinary(_)
+    )
+}
+
 /// Configuration for an index in MemWAL.
 ///
 /// Each variant contains all the configuration needed for that index type.
@@ -139,22 +318,25 @@ impl MemIndexConfig {
 
         // Extract InvertedIndexParams from index_details if available
         let params = if let Some(details_any) = &index_meta.index_details {
-            if let Ok(details) = pbold::InvertedIndexDetails::decode(details_any.value.as_slice()) {
-                InvertedIndexParams::try_from(&details)?
-            } else {
-                InvertedIndexParams::default()
-            }
+            let details = pbold::InvertedIndexDetails::decode(details_any.value.as_slice())
+                .map_err(|err| {
+                    Error::io(format!(
+                        "failed to decode InvertedIndexDetails for MemWAL FTS index '{}': {}",
+                        index_meta.name, err
+                    ))
+                })?;
+            InvertedIndexParams::try_from(&details)?
         } else {
             InvertedIndexParams::default()
         };
         let params = params.format_version(Self::fts_format_version_from_metadata(index_meta)?);
 
-        Ok(Self::Fts(FtsIndexConfig::with_params(
+        Ok(Self::Fts(FtsIndexConfig::try_with_params(
             index_meta.name.clone(),
             field_id,
             column,
             params,
-        )))
+        )?))
     }
 
     /// Create an HNSW vector index config.
@@ -205,8 +387,9 @@ impl MemIndexConfig {
             // the maintained-index path can only write the modern format.
             0 | 1 => Ok(InvertedListFormatVersion::V1),
             2 => Ok(InvertedListFormatVersion::V2),
+            3 => Ok(InvertedListFormatVersion::V3),
             version => Err(Error::invalid_input(format!(
-                "FTS index '{}' has unsupported index_version {}; expected 0, 1, or 2",
+                "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3",
                 index_meta.name, version
             ))),
         }
@@ -237,9 +420,9 @@ impl MemIndexConfig {
 /// Indexes are keyed by index name. Each index stores its field_id for
 /// stable column-to-index resolution (column name → field_id → index).
 ///
-/// The store also carries the MemTable's `max_visible_batch_position`
-/// watermark — the highest batch position that is durable in the WAL and
-/// therefore safe for scanners to read. Scanners snapshot this at plan
+/// The store also carries the MemTable's two cursors: `indexed_count` (what the
+/// index layer has ingested) and `visible_count` (what is indexed *and* durable,
+/// and therefore safe for scanners to read). Scanners snapshot the latter at plan
 /// construction time so every plan keys on a stable MVCC cursor.
 pub struct IndexStore {
     /// BTree indexes keyed by index name. `Arc` so the primary-key BTrees can be
@@ -253,10 +436,23 @@ pub struct IndexStore {
     /// primary key. Queried via [`Self::pk_newest_visible`] (see
     /// [`Self::enable_pk_index`]).
     pk_index: Option<PkIndex>,
-    /// Maximum batch position that is durable in the WAL and therefore
-    /// visible to scanners. Advanced unconditionally after a WAL append
-    /// succeeds; not gated on whether any indexes are configured.
-    max_visible_batch_position: AtomicUsize,
+    /// How many batches of this memtable have been fully indexed. An exclusive
+    /// count: 0 means none.
+    ///
+    /// This has only ever been an *indexed* cursor — it is advanced at the end of
+    /// `insert_batches`, once every index insert for the batch has completed, and
+    /// never before. It was named `max_visible_batch_position` and treated as a
+    /// visibility cursor by five read sites, which is how rows became readable
+    /// before they were durable. Publishing is a separate step, and it is the
+    /// writer's to make — see `visible_count`.
+    indexed_count: AtomicUsize,
+
+    /// The writer's cursors, and this memtable's coordinate within them. `None`
+    /// for a bare `IndexStore` (tests, benches), where visibility is just the
+    /// indexed prefix.
+    ///
+    /// Visibility is **derived, never stored**: see `visible_count`.
+    durability: Option<(Arc<WriterCursors>, usize)>,
     /// Conservative flag set once this memtable has observed any primary-key
     /// rewrite while maintaining a search index. Search planners can push top-k
     /// into HNSW/FTS for append-only PK data, but must switch to
@@ -271,7 +467,8 @@ impl Default for IndexStore {
             hnsw_indexes: HashMap::new(),
             fts_indexes: HashMap::new(),
             pk_index: None,
-            max_visible_batch_position: AtomicUsize::new(0),
+            indexed_count: AtomicUsize::new(0),
+            durability: None,
             pk_has_overrides: AtomicBool::new(false),
         }
     }
@@ -299,10 +496,7 @@ impl std::fmt::Debug for IndexStore {
                     }
                 },
             )
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position.load(Ordering::Acquire),
-            )
+            .field("indexed_count", &self.indexed_count.load(Ordering::Acquire))
             .field(
                 "pk_has_overrides",
                 &self.pk_has_overrides.load(Ordering::Acquire),
@@ -352,8 +546,11 @@ impl IndexStore {
                     registry.hnsw_indexes.insert(c.name.clone(), index);
                 }
                 MemIndexConfig::Fts(c) => {
-                    let index =
-                        FtsMemIndex::with_params(c.field_id, c.column.clone(), c.params.clone());
+                    let index = FtsMemIndex::try_with_params(
+                        c.field_id,
+                        c.column.clone(),
+                        c.params.clone(),
+                    )?;
                     registry.fts_indexes.insert(c.name.clone(), index);
                 }
             }
@@ -452,13 +649,16 @@ impl IndexStore {
         field_id: i32,
         column: String,
         params: InvertedIndexParams,
-    ) {
+    ) -> Result<()> {
         assert!(
             self.pk_index.is_none() || self.pk_is_empty(),
             "FTS indexes must be configured before inserting rows into a PK memtable"
         );
-        self.fts_indexes
-            .insert(name, FtsMemIndex::with_params(field_id, column, params));
+        self.fts_indexes.insert(
+            name,
+            FtsMemIndex::try_with_params(field_id, column, params)?,
+        );
+        Ok(())
     }
 
     /// Maintain a primary-key index so the memtable can answer "newest visible
@@ -516,7 +716,7 @@ impl IndexStore {
     /// BTree (the sidecar dedup index). Single-column emits the typed PK value;
     /// composite emits the order-preserving `Binary` encoded tuple. Empty when
     /// there is no primary key. Row positions line up 1:1 with the forward-
-    /// written data file, so they are the flushed row ids directly.
+    /// written data file, so they are the SSTable row ids directly.
     pub fn pk_training_batches(&self, batch_size: usize) -> Result<Vec<RecordBatch>> {
         match &self.pk_index {
             None => Ok(Vec::new()),
@@ -688,26 +888,25 @@ impl IndexStore {
         let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?;
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update global watermark after all indexes have been updated
+        // Update the indexed prefix after every index has been updated.
         if let Some(bp) = batch_position {
-            self.advance_max_visible_batch_position(bp);
+            self.advance_indexed_count(bp + 1);
         }
 
         Ok(())
     }
 
-    /// Advance the visibility watermark to at least `batch_pos`.
+    /// Advance the indexed prefix to at least `count` batches.
     ///
-    /// The watermark only ever moves forward (idempotent max). The vector
-    /// planner relies on the insert paths setting `pk_has_overrides` before
-    /// calling this method, so any snapshot that can see a PK rewrite also
-    /// observes `pk_has_overrides == true`.
-    pub(crate) fn advance_max_visible_batch_position(&self, batch_pos: usize) {
-        let mut current = self.max_visible_batch_position.load(Ordering::Acquire);
-        while batch_pos > current {
-            match self.max_visible_batch_position.compare_exchange_weak(
+    /// Only ever moves forward (idempotent max). The vector planner relies on the
+    /// insert paths setting `pk_has_overrides` before this is called, so any
+    /// snapshot that can see a PK rewrite also observes `pk_has_overrides == true`.
+    pub(crate) fn advance_indexed_count(&self, count: usize) {
+        let mut current = self.indexed_count.load(Ordering::Acquire);
+        while count > current {
+            match self.indexed_count.compare_exchange_weak(
                 current,
-                batch_pos,
+                count,
                 Ordering::Release,
                 Ordering::Acquire,
             ) {
@@ -717,43 +916,134 @@ impl IndexStore {
         }
     }
 
-    /// Insert multiple batches into all indexes with cross-batch optimization.
+    /// Insert multiple batches into every index.
+    ///
+    /// Above `PARALLEL_INDEX_MIN_ROWS` rows each index runs on its own thread, which
+    /// maximizes parallelism when several indexes are maintained. At or below it they run
+    /// inline on the calling thread: the spawn is one OS thread *per index*, and for a
+    /// handful of rows that costs more than the indexing itself.
+    ///
+    /// Returns a map of index names to their update durations for performance tracking.
     #[instrument(name = "idx_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len()))]
-    pub fn insert_batches(&self, batches: &[StoredBatch]) -> Result<()> {
+    pub fn insert_batches(
+        &self,
+        batches: &[StoredBatch],
+    ) -> Result<std::collections::HashMap<String, std::time::Duration>> {
         if batches.is_empty() {
-            return Ok(());
+            return Ok(std::collections::HashMap::new());
         }
 
         let track_pk_overrides = self.should_track_pk_overrides();
-        // BTree indexes: iterate batches (no cross-batch optimization benefit)
-        for index in self.btree_indexes.values() {
+
+        // One task per index, boxed so the inline and the threaded path drive the very
+        // same closures. Each reports whether it saw an already-present PK.
+        type IndexTask<'a> = Box<dyn Fn() -> Result<bool> + Send + Sync + 'a>;
+        let mut tasks: Vec<(&str, IndexTask<'_>)> = Vec::new();
+
+        for (name, index) in &self.btree_indexes {
             let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
-            let mut had_existing = false;
-            for stored in batches {
-                if track_this_index {
-                    had_existing |=
-                        index.insert_and_report_existing(&stored.data, stored.row_offset)?;
-                } else {
-                    index.insert(&stored.data, stored.row_offset)?;
-                }
-            }
-            self.mark_pk_overrides_if_needed(had_existing);
+            tasks.push((
+                name.as_str(),
+                Box::new(move || {
+                    let mut had_existing = false;
+                    for stored in batches {
+                        if track_this_index {
+                            had_existing |= index
+                                .insert_and_report_existing(&stored.data, stored.row_offset)?;
+                        } else {
+                            index.insert(&stored.data, stored.row_offset)?;
+                        }
+                    }
+                    Ok(had_existing)
+                }),
+            ));
         }
 
-        // HNSW indexes: use batched insert
-        for index in self.hnsw_indexes.values() {
-            index.insert_batches(batches)?;
+        for (name, index) in &self.hnsw_indexes {
+            tasks.push((
+                name.as_str(),
+                Box::new(move || index.insert_batches(batches).map(|_| false)),
+            ));
         }
 
-        // FTS indexes: iterate batches (potential future optimization)
-        for index in self.fts_indexes.values() {
-            for stored in batches {
-                index.insert(&stored.data, stored.row_offset)?;
+        for (name, index) in &self.fts_indexes {
+            tasks.push((
+                name.as_str(),
+                Box::new(move || {
+                    for stored in batches {
+                        index.insert(&stored.data, stored.row_offset)?;
+                    }
+                    Ok(false)
+                }),
+            ));
+        }
+
+        // Keep the raw `Duration` so sub-millisecond timings (the steady state for BTree
+        // updates) survive instead of truncating to 0.
+        let total_rows: usize = batches.iter().map(|b| b.num_rows).sum();
+        let results: Vec<(&str, std::time::Duration, Result<bool>)> =
+            if tasks.len() < 2 || total_rows <= PARALLEL_INDEX_MIN_ROWS {
+                tasks
+                    .iter()
+                    .map(|(name, task)| {
+                        let start = Instant::now();
+                        let result = task();
+                        (*name, start.elapsed(), result)
+                    })
+                    .collect()
+            } else {
+                std::thread::scope(|scope| {
+                    let handles: Vec<_> = tasks
+                        .iter()
+                        .map(|(name, task)| {
+                            let handle = scope.spawn(move || {
+                                let start = Instant::now();
+                                let result = task();
+                                (start.elapsed(), result)
+                            });
+                            (*name, handle)
+                        })
+                        .collect();
+
+                    handles
+                        .into_iter()
+                        .map(|(name, handle)| match handle.join() {
+                            Ok((duration, result)) => (name, duration, result),
+                            Err(_) => (
+                                name,
+                                std::time::Duration::ZERO,
+                                Err(Error::internal(format!("Index '{}' thread panicked", name))),
+                            ),
+                        })
+                        .collect()
+                })
+            };
+
+        // Every task ran to completion whether or not a peer failed (the threaded path
+        // joins all handles unconditionally). Keep the first error; there is no rollback,
+        // so a failure here is terminal for the writer.
+        let mut first_error: Option<Error> = None;
+        let mut had_existing_pk = false;
+        let mut duration_map =
+            std::collections::HashMap::<String, std::time::Duration>::with_capacity(results.len());
+
+        for (name, duration, result) in results {
+            duration_map.insert(name.to_string(), duration);
+            match result {
+                Ok(had_existing) => had_existing_pk |= had_existing,
+                Err(e) if first_error.is_none() => first_error = Some(e),
+                Err(_) => {}
             }
         }
 
-        // Single-column PK aliases a `btree_indexes` entry (maintained above);
-        // a composite PK has its own index, maintained here.
+        if let Some(e) = first_error {
+            return Err(e);
+        }
+        self.mark_pk_overrides_if_needed(had_existing_pk);
+
+        // Single-column PK aliases a `btree_indexes` entry — its task above already
+        // maintained it. A composite PK has its own index; maintain it here before the
+        // watermark advances so the visible prefix is fully indexed.
         let mut had_existing = false;
         for stored in batches {
             had_existing |=
@@ -761,146 +1051,12 @@ impl IndexStore {
         }
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update global watermark to the max batch position
+        // The indexed prefix now covers every batch up to and including the
+        // highest position in this call, so the count is that position plus one.
         let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
-        self.advance_max_visible_batch_position(max_bp);
-
-        Ok(())
-    }
-
-    /// Insert multiple batches into all indexes in parallel.
-    ///
-    /// Each individual index runs in its own thread, regardless of type.
-    /// This maximizes parallelism when multiple indexes are maintained.
-    ///
-    /// This is used during WAL flush to parallelize index updates with WAL I/O.
-    /// Insert batches into all indexes in parallel.
-    ///
-    /// Returns a map of index names to their update durations for performance tracking.
-    #[allow(clippy::print_stderr)]
-    #[instrument(name = "idx_insert_batches_parallel", level = "debug", skip_all, fields(batch_count = batches.len()))]
-    pub fn insert_batches_parallel(
-        &self,
-        batches: &[StoredBatch],
-    ) -> Result<std::collections::HashMap<String, std::time::Duration>> {
-        use std::time::Instant;
-
-        if batches.is_empty() {
-            return Ok(std::collections::HashMap::new());
-        }
-
-        let track_pk_overrides = self.should_track_pk_overrides();
-        // Use std::thread::scope for parallel CPU-bound work
-        std::thread::scope(|scope| {
-            // Each handle returns (index_name, index_type, duration, Result)
-            let mut handles: Vec<(
-                &str,
-                &str,
-                std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result<bool>)>,
-            )> = Vec::new();
-
-            // Spawn a thread for each BTree index
-            for (name, index) in &self.btree_indexes {
-                let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = (|| {
-                        let mut had_existing = false;
-                        for stored in batches {
-                            if track_this_index {
-                                had_existing |= index
-                                    .insert_and_report_existing(&stored.data, stored.row_offset)?;
-                            } else {
-                                index.insert(&stored.data, stored.row_offset)?;
-                            }
-                        }
-                        Ok(had_existing)
-                    })();
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "btree", handle));
-            }
-
-            // Spawn a thread for each HNSW index
-            for (name, index) in &self.hnsw_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = index.insert_batches(batches).map(|_| false);
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "hnsw", handle));
-            }
-
-            // Spawn a thread for each FTS index
-            for (name, index) in &self.fts_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = (|| {
-                        for stored in batches {
-                            index.insert(&stored.data, stored.row_offset)?;
-                        }
-                        Ok(false)
-                    })();
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "fts", handle));
-            }
-
-            // Collect results, log timing, and check for errors. Keep the raw
-            // `Duration` so sub-millisecond timings (the steady-state case for
-            // BTree updates) are preserved instead of getting truncated to 0.
-            let mut first_error: Option<Error> = None;
-            let mut timings: Vec<(&str, &str, std::time::Duration)> = Vec::new();
-            let mut had_existing_pk = false;
-
-            for (name, idx_type, handle) in handles {
-                match handle.join() {
-                    Ok((duration, Ok(had_existing))) => {
-                        timings.push((name, idx_type, duration));
-                        had_existing_pk |= had_existing;
-                    }
-                    Ok((duration, Err(e))) => {
-                        timings.push((name, idx_type, duration));
-                        if first_error.is_none() {
-                            first_error = Some(e);
-                        }
-                    }
-                    Err(_) => {
-                        if first_error.is_none() {
-                            first_error =
-                                Some(Error::internal(format!("Index '{}' thread panicked", name)));
-                        }
-                    }
-                }
-            }
-
-            if let Some(e) = first_error {
-                return Err(e);
-            }
-            self.mark_pk_overrides_if_needed(had_existing_pk);
-
-            let duration_map: std::collections::HashMap<String, std::time::Duration> = timings
-                .into_iter()
-                .map(|(name, _idx_type, duration)| (name.to_string(), duration))
-                .collect();
-
-            // Single-column PK aliases a `btree_indexes` entry — its thread above
-            // already maintained it (and joined). A composite PK has its own
-            // index; maintain it here before the watermark advances so the
-            // visible prefix is fully indexed.
-            let mut had_existing = false;
-            for stored in batches {
-                had_existing |=
-                    self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?;
-            }
-            self.mark_pk_overrides_if_needed(had_existing);
+        self.advance_indexed_count(max_bp + 1);
 
-            // Update global watermark to the max batch position
-            let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
-            self.advance_max_visible_batch_position(max_bp);
-
-            Ok(duration_map)
-        })
+        Ok(duration_map)
     }
 
     /// Get a BTree index by name.
@@ -978,15 +1134,36 @@ impl IndexStore {
         self.btree_indexes.len() + self.hnsw_indexes.len() + self.fts_indexes.len()
     }
 
-    /// Get the visibility watermark (max batch position safe to read).
+    /// How many batches of this memtable have been fully indexed (exclusive
+    /// count; 0 before any batch is indexed).
     ///
-    /// Returns the highest batch position whose data is durable in the WAL
-    /// and therefore visible to scanners. Scanners snapshot this at plan
-    /// construction time so every plan runs against a stable cursor.
+    /// This is the *indexed* cursor, not the visibility watermark: it advances
+    /// once every index insert for a batch completes, regardless of WAL
+    /// durability. Readers must snapshot [`Self::visible_count`], which derives
+    /// what is safe to read from this cursor and the writer's durability cursor.
+    pub fn indexed_count(&self) -> usize {
+        self.indexed_count.load(Ordering::Acquire)
+    }
+
+    /// The prefix of this memtable that readers may see. Snapshot this, never
+    /// `indexed_count`.
     ///
-    /// Returns 0 before any WAL flush has advanced the watermark.
-    pub fn max_visible_batch_position(&self) -> usize {
-        self.max_visible_batch_position.load(Ordering::Acquire)
+    /// Derived on every call from the two cursors rather than cached, so there is
+    /// no published value that can be left stale by a race between the two tasks
+    /// that advance them. A bare `IndexStore` has no writer, so its visible
+    /// prefix is simply what has been indexed.
+    pub fn visible_count(&self) -> usize {
+        let indexed = self.indexed_count();
+        match &self.durability {
+            Some((cursors, global_offset)) => cursors.visible_count(indexed, *global_offset),
+            None => indexed,
+        }
+    }
+
+    /// Bind this memtable's indexes to the writer's cursors. Called once at
+    /// construction, before the memtable is published.
+    pub(crate) fn set_durability(&mut self, cursors: Arc<WriterCursors>, global_offset: usize) {
+        self.durability = Some((cursors, global_offset));
     }
 }
 
@@ -996,6 +1173,7 @@ mod tests {
     use arrow_array::{Int32Array, StringArray};
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use log::warn;
+    use rstest::rstest;
     use std::sync::Arc;
     use uuid::Uuid;
 
@@ -1040,11 +1218,39 @@ mod tests {
         .unwrap()
     }
 
+    fn create_sized_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch {
+        let ids: Vec<i32> = (0..num_rows as i32).map(|i| start_id + i).collect();
+        let names: Vec<String> = ids.iter().map(|id| format!("name-{id}")).collect();
+        let descriptions: Vec<String> = ids.iter().map(|id| format!("hello world {id}")).collect();
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+                Arc::new(StringArray::from(descriptions)),
+            ],
+        )
+        .unwrap()
+    }
+
     fn fts_index_metadata(index_version: i32) -> IndexMetadata {
         let details =
             pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
-        let mut value = Vec::new();
-        details.encode(&mut value).unwrap();
+        fts_index_metadata_with_details(index_version, Some(details))
+    }
+
+    fn fts_index_metadata_with_details(
+        index_version: i32,
+        details: Option<pbold::InvertedIndexDetails>,
+    ) -> IndexMetadata {
+        let index_details = details.map(|details| {
+            let mut value = Vec::new();
+            details.encode(&mut value).unwrap();
+            Arc::new(prost_types::Any {
+                type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(),
+                value,
+            })
+        });
 
         IndexMetadata {
             uuid: Uuid::new_v4(),
@@ -1052,10 +1258,7 @@ mod tests {
             name: "desc_idx".to_string(),
             dataset_version: 1,
             fragment_bitmap: None,
-            index_details: Some(Arc::new(prost_types::Any {
-                type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(),
-                value,
-            })),
+            index_details,
             index_version,
             created_at: None,
             base_id: None,
@@ -1364,6 +1567,7 @@ mod tests {
             (0, InvertedListFormatVersion::V1),
             (1, InvertedListFormatVersion::V1),
             (2, InvertedListFormatVersion::V2),
+            (3, InvertedListFormatVersion::V3),
         ] {
             let config =
                 MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema)
@@ -1386,13 +1590,59 @@ mod tests {
         let arrow_schema = create_test_schema();
         let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
 
-        let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(3), &schema).unwrap_err();
+        let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err();
         assert!(
-            err.to_string().contains("unsupported index_version 3"),
+            err.to_string().contains("unsupported index_version 4"),
             "{err}"
         );
     }
 
+    #[test]
+    fn fts_from_metadata_accepts_v3_with_legacy_block_size() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        for metadata in [
+            fts_index_metadata_with_details(3, None),
+            fts_index_metadata(3),
+        ] {
+            let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap();
+            let MemIndexConfig::Fts(config) = config else {
+                unreachable!("FTS metadata should create an FTS config");
+            };
+            assert_eq!(
+                config.params.resolved_format_version(),
+                InvertedListFormatVersion::V3
+            );
+            assert_eq!(config.params.posting_block_size(), 128);
+        }
+    }
+
+    #[test]
+    fn fts_from_metadata_accepts_v3_with_256_block_size() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let details = pbold::InvertedIndexDetails::try_from(&params).unwrap();
+
+        let config = MemIndexConfig::fts_from_metadata(
+            &fts_index_metadata_with_details(3, Some(details)),
+            &schema,
+        )
+        .unwrap();
+
+        match config {
+            MemIndexConfig::Fts(config) => {
+                assert_eq!(
+                    config.params.resolved_format_version(),
+                    InvertedListFormatVersion::V3
+                );
+                assert_eq!(config.params.posting_block_size(), 256);
+            }
+            _ => unreachable!("fts metadata should create an FTS config"),
+        }
+    }
+
     #[test]
     fn test_from_configs() {
         let configs = vec![
@@ -1417,8 +1667,124 @@ mod tests {
         assert!(registry.get_fts_by_field_id(2).is_some());
     }
 
+    fn vector_schema() -> Arc<ArrowSchema> {
+        Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("description", DataType::Utf8, true),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+            Field::new(
+                "f64_vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), 4),
+                true,
+            ),
+        ]))
+    }
+
+    /// Every index config that would fail *deterministically* on insert must be
+    /// rejected at open instead. Such a config also fails on WAL replay, so once
+    /// a row is durable the shard could never reopen — poison-and-replay would
+    /// not terminate.
+    #[rstest]
+    #[case::btree_ok(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 0, column: "id".into(),
+    }), None)]
+    #[case::btree_missing_column(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 9, column: "nope".into(),
+    }), Some("not in the shard schema"))]
+    // Column exists, but its field_id names a *different* column ("id" is 0, not 1).
+    #[case::btree_field_id_column_mismatch(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 1, column: "id".into(),
+    }), Some("has field_id 0"))]
+    #[case::fts_ok(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 1, "description".into(),
+    )), None)]
+    #[case::fts_non_utf8(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 0, "id".into(),
+    )), Some("requires a Utf8, LargeUtf8, or Utf8View column"))]
+    #[case::fts_missing_column(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 9, "nope".into(),
+    )), Some("not in the shard schema"))]
+    #[case::hnsw_ok(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 2, "vector".into(), DistanceType::L2,
+    ))), None)]
+    #[case::hnsw_not_a_vector(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 0, "id".into(), DistanceType::L2,
+    ))), Some("requires a FixedSizeList<Float32> column"))]
+    #[case::hnsw_wrong_item_type(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 3, "f64_vector".into(), DistanceType::L2,
+    ))), Some("item type Float64"))]
+    #[case::hnsw_missing_column(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 9, "nope".into(), DistanceType::L2,
+    ))), Some("not in the shard schema"))]
+    fn test_validate_index_configs(
+        #[case] config: MemIndexConfig,
+        #[case] expected_error: Option<&str>,
+    ) {
+        let schema = vector_schema();
+        let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap();
+        let result = validate_index_configs(&[config], &schema, &lance_schema, &[]);
+        match expected_error {
+            None => result.expect("valid config must pass validation"),
+            Some(fragment) => {
+                let message = result
+                    .expect_err("invalid config must be rejected")
+                    .to_string();
+                assert!(
+                    message.contains(fragment),
+                    "error must explain the mismatch; wanted {fragment:?}, got {message:?}"
+                );
+            }
+        }
+    }
+
+    /// A composite PK builds an order-preserving encoded key, so its columns must
+    /// be encodable. A single-column PK aliases a BTree entry, which accepts any
+    /// type — so it must *not* be rejected here.
     #[test]
-    fn test_index_store_max_visible_batch_position() {
+    fn test_validate_composite_pk_column_types() {
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new(
+                "coords",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+        let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap();
+
+        validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "name".into()])
+            .expect("Int32 + Utf8 composite PK must be encodable");
+
+        let err =
+            validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "coords".into()])
+                .expect_err("a FixedSizeList PK column has no order-preserving encoding");
+        assert!(
+            err.to_string().contains("order-preserving key encoding"),
+            "error must name the reason, got {err}"
+        );
+
+        // A single-column PK of the same type is fine: it aliases a BTree.
+        validate_index_configs(&[], &schema, &lance_schema, &["coords".into()])
+            .expect("single-column PK aliases a BTree and accepts any type");
+
+        // But every PK column must exist. A single-column PK naming an absent
+        // column is rejected here, not left to fail deterministically on every
+        // later index build and WAL replay.
+        let err = validate_index_configs(&[], &schema, &lance_schema, &["missing".into()])
+            .expect_err("a single-column PK on an absent column must be rejected");
+        assert!(
+            err.to_string().contains("not in the shard schema"),
+            "error must name the missing column, got {err}"
+        );
+    }
+
+    #[test]
+    fn test_index_store_indexed_count() {
         let schema = create_test_schema();
         let mut registry = IndexStore::new();
 
@@ -1427,7 +1793,7 @@ mod tests {
         registry.add_fts("desc_idx".to_string(), 2, "description".to_string());
 
         // Initial watermark should be 0 (no data indexed yet)
-        assert_eq!(registry.max_visible_batch_position(), 0);
+        assert_eq!(registry.indexed_count(), 0);
 
         // Insert with batch position tracking
         let batch = create_test_batch(&schema, 0);
@@ -1435,20 +1801,54 @@ mod tests {
             .insert_with_batch_position(&batch, 0, Some(5))
             .unwrap();
 
-        // Now watermark should be 5
-        assert_eq!(registry.max_visible_batch_position(), 5);
+        // Indexing batch position 5 means the prefix [0, 6) is indexed.
+        assert_eq!(registry.indexed_count(), 6);
 
         // Insert with higher batch position
         registry
             .insert_with_batch_position(&batch, 3, Some(10))
             .unwrap();
 
-        // Watermark should advance to 10
-        assert_eq!(registry.max_visible_batch_position(), 10);
+        // Advances to cover batch position 10.
+        assert_eq!(registry.indexed_count(), 11);
 
-        // Insert without batch position shouldn't change watermark
+        // Insert without batch position shouldn't change the cursor
         registry.insert(&batch, 6).unwrap();
-        assert_eq!(registry.max_visible_batch_position(), 10);
+        assert_eq!(registry.indexed_count(), 11);
+    }
+
+    /// `insert_batches` picks the inline or the threaded path by row count, so
+    /// exercise both and assert they leave the same index state: every row indexed
+    /// exactly once, in every index, with a timing reported for each.
+    #[rstest]
+    #[case::inline(8)]
+    #[case::threaded(PARALLEL_INDEX_MIN_ROWS + 64)]
+    fn test_insert_batches_indexes_every_row_once(#[case] num_rows: usize) {
+        let schema = create_test_schema();
+        let mut registry = IndexStore::new();
+        registry.add_btree("id_idx".to_string(), 0, "id".to_string());
+        registry.add_fts("desc_idx".to_string(), 2, "description".to_string());
+
+        let batch = create_sized_batch(&schema, 0, num_rows);
+        let durations = registry
+            .insert_batches(&[StoredBatch::new(batch, 0, 2)])
+            .unwrap();
+
+        assert_eq!(durations.len(), 2, "expected one timing per index");
+        assert!(durations.contains_key("id_idx"));
+        assert!(durations.contains_key("desc_idx"));
+
+        let btree = registry.get_btree("id_idx").unwrap();
+        for id in 0..num_rows as i32 {
+            let positions = btree.get(&ScalarValue::Int32(Some(id)));
+            assert_eq!(
+                positions.len(),
+                1,
+                "id={id} should be indexed exactly once, got {positions:?}"
+            );
+        }
+        assert_eq!(registry.get_fts("desc_idx").unwrap().doc_count(), num_rows);
+        assert_eq!(registry.indexed_count(), 3);
     }
 
     #[test]
diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 61c797db041..24d8cd8cbc3 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -59,8 +59,8 @@ use fst::{Map, Streamer};
 use lance_bitpacking::{BitPacker, BitPacker4x};
 use lance_core::{Error, Result};
 use lance_index::scalar::InvertedIndexParams;
-use lance_index::scalar::inverted::query::Operator;
-use lance_index::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
+use lance_index::scalar::inverted::query::{Operator, Tokens};
+use lance_index::scalar::inverted::tokenizer::document_tokenizer::{DocType, LanceTokenizer};
 use lance_index::scalar::inverted::{DocSet, MemBM25Scorer, Scorer, TokenSet};
 use lance_tokenizer::TokenStream;
 use rayon::prelude::*;
@@ -371,6 +371,41 @@ fn char_prefix(term: &str, prefix_length: u32) -> &str {
         .unwrap_or(term)
 }
 
+fn query_tokens_to_vec(tokens: &Tokens) -> Vec<String> {
+    (0..tokens.len())
+        .map(|idx| tokens.get_token(idx).to_string())
+        .collect()
+}
+
+fn has_grouped_positions(tokens: &Tokens) -> bool {
+    let mut seen = HashSet::new();
+    (0..tokens.len()).any(|idx| !seen.insert(tokens.position(idx)))
+}
+
+fn query_position_groups(tokens: &Tokens) -> Vec<Vec<String>> {
+    let mut groups = Vec::new();
+    let mut current_position = None;
+    for idx in 0..tokens.len() {
+        let position = tokens.position(idx);
+        if current_position != Some(position) {
+            current_position = Some(position);
+            groups.push(Vec::new());
+        }
+        let group = groups
+            .last_mut()
+            .expect("a group should exist after pushing for position");
+        let token = tokens.get_token(idx).to_string();
+        if !group.contains(&token) {
+            group.push(token);
+        }
+    }
+    groups
+}
+
+fn position_groups_to_tokens(groups: &[Vec<String>]) -> Vec<String> {
+    groups.iter().flatten().cloned().collect()
+}
+
 /// Builder for constructing Boolean queries.
 #[derive(Debug, Clone, Default)]
 pub struct BooleanQueryBuilder {
@@ -543,6 +578,7 @@ struct BatchMeta {
     batch_position: usize,
     row_offset: u64,
     /// `doc_lengths[i]` is the token count of the row at `row_offset + i`.
+    /// Zero entries preserve row-position alignment but are not documents.
     doc_lengths: Vec<u32>,
     rows: u32,
 }
@@ -639,7 +675,7 @@ struct Snapshot {
     /// visible_count` for any snapshot the writer has stored (each publish
     /// appends one entry and bumps `visible_count`).
     batches: BatchLog,
-    /// `Σ batches[i].rows` for `i < visible_count`.
+    /// Number of non-zero-token documents for `i < visible_count`.
     cumulative_doc_count: u64,
     /// `Σ batches[i].doc_lengths.iter().sum()` for `i < visible_count`.
     cumulative_total_tokens: u64,
@@ -811,6 +847,7 @@ impl TailIndex {
         term_builders: FxHashMap<Arc<str>, BatchTermBuilder>,
         with_position: bool,
     ) {
+        let doc_count = doc_lengths.iter().filter(|&&len| len > 0).count() as u64;
         let mut cache = self
             .writer_term_cache
             .lock()
@@ -840,7 +877,7 @@ impl TailIndex {
         self.snapshot.store(Arc::new(Snapshot {
             visible_count: cur.visible_count + 1,
             batches: cur.batches.pushed(new_meta),
-            cumulative_doc_count: cur.cumulative_doc_count + rows as u64,
+            cumulative_doc_count: cur.cumulative_doc_count + doc_count,
             cumulative_total_tokens: cur.cumulative_total_tokens + total_tokens,
         }));
     }
@@ -960,10 +997,20 @@ impl FtsMemIndex {
 
     /// Create a new FTS index with custom tokenizer parameters.
     pub fn with_params(field_id: i32, column_name: String, params: InvertedIndexParams) -> Self {
-        let pool = TokenizerPool::new(&params, Self::DEFAULT_TOKENIZER_POOL_CAP)
-            .expect("Failed to build tokenizer");
+        Self::try_with_params(field_id, column_name, params)
+            .expect("invalid MemWAL FTS index parameters")
+    }
+
+    /// Try to create a new FTS index with custom tokenizer parameters.
+    pub fn try_with_params(
+        field_id: i32,
+        column_name: String,
+        params: InvertedIndexParams,
+    ) -> Result<Self> {
+        params.validate_format_version()?;
+        let pool = TokenizerPool::new(&params, Self::DEFAULT_TOKENIZER_POOL_CAP)?;
         let writer_tokenizer = pool.template.box_clone();
-        Self {
+        Ok(Self {
             field_id,
             column_name,
             params,
@@ -972,7 +1019,7 @@ impl FtsMemIndex {
             state: ArcSwap::from(IndexState::empty()),
             freeze_threshold_rows: Self::DEFAULT_FREEZE_THRESHOLD_ROWS,
             merge: Arc::new(Mutex::new(None)),
-        }
+        })
     }
 
     /// Override the tail freeze threshold (docs) — the analogue of Lucene's
@@ -1084,27 +1131,31 @@ impl FtsMemIndex {
         self.insert_batch(batch, row_offset)
     }
 
+    fn checked_token_position(row_position: RowPosition, token_position: usize) -> Result<u32> {
+        u32::try_from(token_position).map_err(|_| {
+            Error::invalid_input(format!(
+                "token position overflow for row_position={row_position}: token_position={token_position}"
+            ))
+        })
+    }
+
     fn insert_batch(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> {
         let st = self.state.load_full();
-        let batch_position = st.tail.next_position();
 
+        // A missing column is a config error, not an empty document: silently
+        // appending an empty batch would leave the index empty forever while the
+        // shard reported healthy. BTree and HNSW both reject this; so do we.
+        // `validate_index_configs` catches it at open, so reaching here means a
+        // batch got past the memtable's schema-equality gate.
         let Some(col_idx) = batch
             .schema()
             .column_with_name(&self.column_name)
             .map(|(idx, _)| idx)
         else {
-            // Column missing: nothing to index, but publish an empty batch so
-            // the tail's visibility counters keep up with the writer.
-            st.tail.append_batch(
-                batch_position,
-                row_offset,
-                batch.num_rows() as u32,
-                vec![0; batch.num_rows()],
-                0,
-                FxHashMap::default(),
-                self.params.has_positions(),
-            );
-            return Ok(());
+            return Err(Error::invalid_input(format!(
+                "FTS index column '{}' is not in the inserted batch schema",
+                self.column_name
+            )));
         };
 
         let column = batch.column(col_idx);
@@ -1133,8 +1184,8 @@ impl FtsMemIndex {
 
             if let Some(text) = text_opt {
                 let mut stream = tokenizer.token_stream_for_doc(text);
-                let mut position: u32 = 0;
                 while let Some(tok) = stream.next() {
+                    let position = Self::checked_token_position(row_position, tok.position)?;
                     let term = tok.text.as_str();
                     // One hash lookup per token: extend the term's builder, or
                     // intern its `Arc<str>` once on first sight this batch.
@@ -1146,7 +1197,6 @@ impl FtsMemIndex {
                             BatchTermBuilder::with_first(row_position, position),
                         );
                     }
-                    position += 1;
                     doc_token_count += 1;
                 }
             }
@@ -1155,10 +1205,15 @@ impl FtsMemIndex {
             total_tokens += doc_token_count as u64;
         }
 
+        if total_tokens == 0 {
+            return Ok(());
+        }
+
         // Drop the tokenizer guard before publishing so we don't hold it
         // across the snapshot install.
         drop(tok_guard);
 
+        let batch_position = st.tail.next_position();
         st.tail.append_batch(
             batch_position,
             row_offset,
@@ -1267,7 +1322,7 @@ impl FtsMemIndex {
     /// use `search_with_options` for sorted/limited output.
     pub fn search(&self, term: &str) -> Vec<FtsEntry> {
         let st = self.state.load_full();
-        let tokens = self.tokenize_for_search(term);
+        let tokens = self.analyze_for_search(term);
         self.search_match(&st, &tokens, Operator::Or, None, true, true)
     }
 
@@ -1275,7 +1330,7 @@ impl FtsMemIndex {
     /// `slop` intervening tokens between consecutive query tokens.
     pub fn search_phrase(&self, phrase: &str, slop: u32) -> Vec<FtsEntry> {
         let st = self.state.load_full();
-        let tokens = self.tokenize_for_search(phrase);
+        let tokens = self.analyze_for_search(phrase);
         self.search_phrase_tokens(&st, &tokens, slop, true)
     }
 
@@ -1322,6 +1377,22 @@ impl FtsMemIndex {
     /// shared rising threshold (instead of every partition cold-starting).
     /// Without a limit, an exact O(matches) scan across partitions + tail.
     fn search_match(
+        &self,
+        st: &IndexState,
+        query_tokens: &Tokens,
+        operator: Operator,
+        limit: Option<usize>,
+        include_tail: bool,
+        tail_skip: bool,
+    ) -> Vec<FtsEntry> {
+        if operator == Operator::And && has_grouped_positions(query_tokens) {
+            return self.search_grouped_and(st, query_tokens, limit, include_tail, tail_skip);
+        }
+        let tokens = query_tokens_to_vec(query_tokens);
+        self.search_match_strings(st, &tokens, operator, limit, include_tail, tail_skip)
+    }
+
+    fn search_match_strings(
         &self,
         st: &IndexState,
         tokens: &[String],
@@ -1390,19 +1461,82 @@ impl FtsMemIndex {
         }
     }
 
+    fn search_grouped_and(
+        &self,
+        st: &IndexState,
+        query_tokens: &Tokens,
+        limit: Option<usize>,
+        include_tail: bool,
+        tail_skip: bool,
+    ) -> Vec<FtsEntry> {
+        let mut result_map: Option<HashMap<RowPosition, f32>> = None;
+        for group in query_position_groups(query_tokens) {
+            let group_results =
+                self.search_match_strings(st, &group, Operator::Or, None, include_tail, tail_skip);
+            let group_map = group_results
+                .into_iter()
+                .map(|entry| (entry.row_position, entry.score))
+                .collect::<HashMap<_, _>>();
+            let Some(current) = result_map.as_mut() else {
+                result_map = Some(group_map);
+                continue;
+            };
+            current.retain(|row_position, score| {
+                if let Some(group_score) = group_map.get(row_position) {
+                    *score += group_score;
+                    true
+                } else {
+                    false
+                }
+            });
+            if current.is_empty() {
+                return Vec::new();
+            }
+        }
+
+        let mut results = result_map
+            .unwrap_or_default()
+            .into_iter()
+            .map(|(row_position, score)| FtsEntry {
+                row_position,
+                score,
+            })
+            .collect::<Vec<_>>();
+        if let Some(limit) = limit {
+            results.sort_by(|a, b| {
+                b.score
+                    .partial_cmp(&a.score)
+                    .unwrap_or(std::cmp::Ordering::Equal)
+            });
+            results.truncate(limit);
+        }
+        results
+    }
+
     fn search_phrase_tokens(
         &self,
         st: &IndexState,
-        tokens: &[String],
+        query_tokens: &Tokens,
         slop: u32,
         include_tail: bool,
     ) -> Vec<FtsEntry> {
-        if tokens.is_empty() {
+        if query_tokens.is_empty() {
             return Vec::new();
         }
-        if tokens.len() == 1 {
+        let groups = query_position_groups(query_tokens);
+        if groups.is_empty() {
+            return Vec::new();
+        }
+        if groups.len() == 1 {
             // A single-token phrase reduces to a regular term search.
-            return self.search_match(st, tokens, Operator::Or, None, include_tail, true);
+            return self.search_match_strings(
+                st,
+                &groups[0],
+                Operator::Or,
+                None,
+                include_tail,
+                true,
+            );
         }
         // A multi-token phrase needs token positions; without them (the index
         // was built `with_position = false`) phrase search is unsupported, as
@@ -1410,24 +1544,40 @@ impl FtsMemIndex {
         if !self.params.has_positions() {
             return Vec::new();
         }
+        let has_grouped_terms = groups.iter().any(|group| group.len() > 1);
+        let tokens = position_groups_to_tokens(&groups);
         let tail_snap = st.tail.snapshot();
         let scan_tail = include_tail && tail_snap.visible_count > 0;
-        let scorer = build_scorer(st, &tail_snap, tokens, include_tail);
+        let scorer = build_scorer(st, &tail_snap, &tokens, include_tail);
         if scorer.num_docs() == 0 {
             return Vec::new();
         }
         let mut results = Vec::new();
         for p in st.partitions.iter() {
-            results.extend(p.search_phrase(tokens, slop, &scorer));
+            if has_grouped_terms {
+                results.extend(p.search_phrase_groups(&groups, slop, &scorer));
+            } else {
+                results.extend(p.search_phrase(&tokens, slop, &scorer));
+            }
         }
         if scan_tail {
-            results.extend(phrase_search_tail(
-                &tail_snap,
-                &st.tail.terms,
-                tokens,
-                slop,
-                &scorer,
-            ));
+            if has_grouped_terms {
+                results.extend(phrase_search_tail_groups(
+                    &tail_snap,
+                    &st.tail.terms,
+                    &groups,
+                    slop,
+                    &scorer,
+                ));
+            } else {
+                results.extend(phrase_search_tail(
+                    &tail_snap,
+                    &st.tail.terms,
+                    &tokens,
+                    slop,
+                    &scorer,
+                ));
+            }
         }
         results
     }
@@ -1464,7 +1614,7 @@ impl FtsMemIndex {
         if expanded.is_empty() {
             return Vec::new();
         }
-        self.search_match(st, &expanded, Operator::Or, None, include_tail, true)
+        self.search_match_strings(st, &expanded, Operator::Or, None, include_tail, true)
     }
 
     /// Expand `term` against the term dictionaries of every partition (and the
@@ -1556,14 +1706,14 @@ impl FtsMemIndex {
                 operator,
                 boost,
             } => {
-                let tokens = self.tokenize_for_search(query);
+                let tokens = self.analyze_for_search(query);
                 let mut results =
                     self.search_match(st, &tokens, *operator, limit, include_tail, tail_skip);
                 apply_boost(&mut results, *boost);
                 results
             }
             FtsQueryExpr::Phrase { query, slop, boost } => {
-                let tokens = self.tokenize_for_search(query);
+                let tokens = self.analyze_for_search(query);
                 let mut results = self.search_phrase_tokens(st, &tokens, *slop, include_tail);
                 apply_boost(&mut results, *boost);
                 results
@@ -1736,13 +1886,19 @@ impl FtsMemIndex {
     }
 
     fn tokenize_for_search(&self, text: &str) -> Vec<String> {
+        query_tokens_to_vec(&self.analyze_for_search(text))
+    }
+
+    fn analyze_for_search(&self, text: &str) -> Tokens {
         let mut tok = PooledTokenizer::new(&self.tokenizer_pool);
         let mut stream = tok.get_mut().token_stream_for_search(text);
-        let mut out = Vec::new();
+        let mut tokens = Vec::new();
+        let mut positions = Vec::new();
         while let Some(t) = stream.next() {
-            out.push(t.text.clone());
+            tokens.push(t.text.clone());
+            positions.push(t.position as u32);
         }
-        out
+        Tokens::with_positions(tokens, positions, DocType::Text)
     }
 
     // ------------------------------------------------------------------
@@ -1752,8 +1908,9 @@ impl FtsMemIndex {
     /// Export the in-memory FTS index to an `InnerBuilder` ready to be
     /// written to disk.
     ///
-    /// Doc row positions are kept in insert order to match the forward-written
-    /// flush data file 1:1. `total_rows` is used only to validate positions.
+    /// Documents are kept in insert order and retain their positions in the
+    /// forward-written flush data file; zero-token rows are omitted.
+    /// `total_rows` is used only to validate positions.
     pub fn to_index_builder(
         &self,
         partition_id: u64,
@@ -1764,6 +1921,7 @@ impl FtsMemIndex {
 
         let st = self.state.load_full();
         let with_position = self.params.has_positions();
+        let block_size = self.params.posting_block_size();
         let format_version = self.params.resolved_format_version();
         let posting_tail_codec = format_version.posting_tail_codec();
         let total_rows_u64 = total_rows as u64;
@@ -1779,20 +1937,24 @@ impl FtsMemIndex {
         let tail_snap = st.tail.snapshot();
         for batch in tail_snap.batches.iter().take(tail_snap.visible_count) {
             for i in 0..batch.rows as usize {
-                all_docs.push((batch.row_offset + i as u64, batch.doc_lengths[i]));
+                let num_tokens = batch.doc_lengths[i];
+                if num_tokens > 0 {
+                    all_docs.push((batch.row_offset + i as u64, num_tokens));
+                }
             }
         }
         if all_docs.is_empty() {
-            return Ok(InnerBuilder::new_with_format_version(
+            return Ok(InnerBuilder::new_with_format_version_and_block_size(
                 partition_id,
                 with_position,
                 Default::default(),
                 format_version,
+                block_size,
             ));
         }
 
-        // Step 2: assign doc_ids in ascending insert-position order, so the
-        // stored row positions line up 1:1 with the forward-written data file.
+        // Step 2: assign doc_ids in ascending insert-position order while
+        // preserving each document's position in the forward-written data file.
         let mut entries: Vec<(u64, u32)> = Vec::with_capacity(all_docs.len());
         for (original, num_tokens) in &all_docs {
             if *original >= total_rows_u64 {
@@ -1873,10 +2035,13 @@ impl FtsMemIndex {
             docs_for_term.sort_by_key(|(doc_id, _, _)| *doc_id);
             let token_id = tokens.add(token) as usize;
             debug_assert_eq!(token_id, posting_lists.len());
-            posting_lists.push(PostingListBuilder::new_with_posting_tail_codec(
-                with_position,
-                posting_tail_codec,
-            ));
+            posting_lists.push(
+                PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                    with_position,
+                    posting_tail_codec,
+                    block_size,
+                ),
+            );
             let plb = &mut posting_lists[token_id];
             for (doc_id, freq, pos) in docs_for_term {
                 let recorder = if with_position {
@@ -1888,11 +2053,12 @@ impl FtsMemIndex {
             }
         }
 
-        let mut builder = InnerBuilder::new_with_format_version(
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
             partition_id,
             with_position,
             Default::default(),
             format_version,
+            block_size,
         );
         builder.set_tokens(tokens);
         builder.set_docs(docs);
@@ -2264,6 +2430,126 @@ fn phrase_search_tail(
     results
 }
 
+#[derive(Default, Clone)]
+struct PhraseGroupDoc {
+    positions: Vec<u32>,
+    score: f32,
+}
+
+struct PhraseCandidate {
+    positions_by_group: Vec<Vec<u32>>,
+    score: f32,
+}
+
+fn merge_phrase_group<K>(
+    candidates: &mut Option<HashMap<K, PhraseCandidate>>,
+    group_idx: usize,
+    group_count: usize,
+    group_docs: HashMap<K, PhraseGroupDoc>,
+) -> bool
+where
+    K: Copy + Eq + std::hash::Hash,
+{
+    let Some(current) = candidates.as_mut() else {
+        *candidates = Some(
+            group_docs
+                .into_iter()
+                .map(|(doc, group_doc)| {
+                    let mut positions_by_group = vec![Vec::new(); group_count];
+                    positions_by_group[group_idx] = group_doc.positions;
+                    (
+                        doc,
+                        PhraseCandidate {
+                            positions_by_group,
+                            score: group_doc.score,
+                        },
+                    )
+                })
+                .collect(),
+        );
+        return true;
+    };
+
+    current.retain(|doc, candidate| {
+        if let Some(group_doc) = group_docs.get(doc) {
+            candidate.positions_by_group[group_idx] = group_doc.positions.clone();
+            candidate.score += group_doc.score;
+            true
+        } else {
+            false
+        }
+    });
+    !current.is_empty()
+}
+
+fn phrase_search_tail_groups(
+    snap: &Snapshot,
+    terms: &SkipMap<Arc<str>, Arc<ArcSwap<TermSlice>>>,
+    groups: &[Vec<String>],
+    slop: u32,
+    scorer: &MemBM25Scorer,
+) -> Vec<FtsEntry> {
+    let mut candidates: Option<HashMap<RowPosition, PhraseCandidate>> = None;
+    for (group_idx, group) in groups.iter().enumerate() {
+        let group_docs = tail_phrase_group_docs(snap, terms, group, scorer);
+        if group_docs.is_empty()
+            || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs)
+        {
+            return Vec::new();
+        }
+    }
+
+    candidates
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop))
+        .map(|(row_position, candidate)| FtsEntry {
+            row_position,
+            score: candidate.score,
+        })
+        .collect()
+}
+
+fn tail_phrase_group_docs(
+    snap: &Snapshot,
+    terms: &SkipMap<Arc<str>, Arc<ArcSwap<TermSlice>>>,
+    group: &[String],
+    scorer: &MemBM25Scorer,
+) -> HashMap<RowPosition, PhraseGroupDoc> {
+    let mut docs: HashMap<RowPosition, PhraseGroupDoc> = HashMap::new();
+    for token in group {
+        let Some(entry) = terms.get(token.as_str()) else {
+            continue;
+        };
+        let qw = scorer.query_weight(token);
+        let slice = entry.value().load_full();
+        for chunk in slice.chunks() {
+            if chunk.batch_position >= snap.visible_count {
+                continue;
+            }
+            let Some(meta) = snap.batch_for(chunk.batch_position) else {
+                continue;
+            };
+            let Some(positions) = &chunk.positions else {
+                continue;
+            };
+            for (i, &row_position) in chunk.row_positions.iter().enumerate() {
+                let entry = docs.entry(row_position).or_default();
+                entry
+                    .positions
+                    .extend_from_slice(positions.doc_positions(i));
+                let dl = meta.dl(row_position).unwrap_or(1);
+                entry.score += qw * scorer.doc_weight(chunk.frequencies[i], dl);
+            }
+        }
+    }
+    for doc in docs.values_mut() {
+        doc.positions.sort_unstable();
+        doc.positions.dedup();
+    }
+    docs
+}
+
 fn phrase_matches<T: AsRef<[u32]>>(positions: &[T], slop: u32) -> bool {
     if positions.is_empty() {
         return false;
@@ -2332,12 +2618,23 @@ impl FtsIndexConfig {
         column: String,
         params: InvertedIndexParams,
     ) -> Self {
-        Self {
+        Self::try_with_params(name, field_id, column, params)
+            .expect("invalid MemWAL FTS index config parameters")
+    }
+
+    pub fn try_with_params(
+        name: String,
+        field_id: i32,
+        column: String,
+        params: InvertedIndexParams,
+    ) -> Result<Self> {
+        params.validate_format_version()?;
+        Ok(Self {
             name,
             field_id,
             column,
             params,
-        }
+        })
     }
 }
 
@@ -2970,7 +3267,11 @@ impl Partition {
         for batch in snap.batches.iter().take(snap.visible_count) {
             for i in 0..batch.rows as usize {
                 let rp = batch.row_offset + i as u64;
-                let doc_id = docs.append(rp, batch.doc_lengths[i]);
+                let num_tokens = batch.doc_lengths[i];
+                if num_tokens == 0 {
+                    continue;
+                }
+                let doc_id = docs.append(rp, num_tokens);
                 pos_to_doc.insert(rp, doc_id);
             }
         }
@@ -3313,6 +3614,62 @@ impl Partition {
         }
         results
     }
+
+    fn search_phrase_groups(
+        &self,
+        groups: &[Vec<String>],
+        slop: u32,
+        scorer: &MemBM25Scorer,
+    ) -> Vec<FtsEntry> {
+        let mut candidates: Option<HashMap<u32, PhraseCandidate>> = None;
+        for (group_idx, group) in groups.iter().enumerate() {
+            let group_docs = self.phrase_group_docs(group, scorer);
+            if group_docs.is_empty()
+                || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs)
+            {
+                return Vec::new();
+            }
+        }
+
+        candidates
+            .unwrap_or_default()
+            .into_iter()
+            .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop))
+            .map(|(doc, candidate)| FtsEntry {
+                row_position: self.docs.row_id(doc),
+                score: candidate.score,
+            })
+            .collect()
+    }
+
+    fn phrase_group_docs(
+        &self,
+        group: &[String],
+        scorer: &MemBM25Scorer,
+    ) -> HashMap<u32, PhraseGroupDoc> {
+        let mut docs: HashMap<u32, PhraseGroupDoc> = HashMap::new();
+        for token in group {
+            let Some(term_id) = self.term_id(token) else {
+                continue;
+            };
+            let qw = scorer.query_weight(token);
+            let mut cursor = PostingCursor::new(self, term_id);
+            while let Some(doc) = cursor.cursor_doc() {
+                let positions = cursor.positions().to_vec();
+                let freq = cursor.freq();
+                let dl = self.docs.num_tokens(doc);
+                let entry = docs.entry(doc).or_default();
+                entry.positions.extend_from_slice(&positions);
+                entry.score += qw * scorer.doc_weight(freq, dl);
+                cursor.advance();
+            }
+        }
+        for doc in docs.values_mut() {
+            doc.positions.sort_unstable();
+            doc.positions.dedup();
+        }
+        docs
+    }
 }
 
 /// A scored MemTable row, ordered by score then row position (`total_cmp`,
@@ -3652,6 +4009,182 @@ mod tests {
         assert!(entries.is_empty());
     }
 
+    #[test]
+    fn test_code_analyzer_and_query_uses_position_alternatives() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::with_params(
+            1,
+            "description".to_string(),
+            InvertedIndexParams::code().split_identifiers(true),
+        );
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1])),
+                Arc::new(StringArray::from(vec!["get user name", "getUserName"])),
+            ],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+
+        let query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And);
+        let mut rows = index
+            .search_query(&query)
+            .into_iter()
+            .map(|entry| entry.row_position)
+            .collect::<Vec<_>>();
+        rows.sort_unstable();
+        assert_eq!(rows, vec![0, 1]);
+    }
+
+    fn tail_positions_for(index: &FtsMemIndex, term: &str, row_position: RowPosition) -> Vec<u32> {
+        let st = index.state.load_full();
+        let snap = st.tail.snapshot();
+        let entry = st.tail.terms.get(term).expect("term should be indexed");
+        let slice = entry.value().load();
+        for chunk in slice.chunks() {
+            if chunk.batch_position >= snap.visible_count {
+                continue;
+            }
+            if let Ok(doc_idx) = chunk.row_positions.binary_search(&row_position) {
+                return chunk
+                    .positions
+                    .as_ref()
+                    .expect("test index stores positions")
+                    .doc_positions(doc_idx)
+                    .to_vec();
+            }
+        }
+        panic!("term {term} should be present in row {row_position}");
+    }
+
+    #[test]
+    fn test_code_analyzer_phrase_uses_token_positions_and_alternatives() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::with_params(
+            1,
+            "description".to_string(),
+            InvertedIndexParams::code()
+                .with_position(true)
+                .split_identifiers(true),
+        );
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "get user name",
+                    "getUserName",
+                    "get user",
+                    "get fast user name",
+                ])),
+            ],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+
+        assert_eq!(tail_positions_for(&index, "getusername", 1), vec![0]);
+        assert_eq!(tail_positions_for(&index, "get", 1), vec![0]);
+        assert_eq!(tail_positions_for(&index, "user", 1), vec![1]);
+        assert_eq!(tail_positions_for(&index, "name", 1), vec![2]);
+
+        let query = FtsQueryExpr::phrase("getUserName");
+        assert_eq!(rows(index.search_phrase("getUserName", 0)), vec![0, 1]);
+        assert_eq!(rows(index.search_query(&query)), vec![0, 1]);
+
+        index.flush();
+        let partition_only = SearchOptions::new().with_include_tail(false);
+        assert_eq!(
+            rows(index.search_with_options(&query, partition_only)),
+            vec![0, 1]
+        );
+    }
+
+    #[test]
+    fn test_zero_token_documents_are_skipped_across_memwal_paths() {
+        let params =
+            InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English)
+                .remove_stop_words(true)
+                .stem(false)
+                .max_token_length(Some(6));
+        let schema = create_test_schema();
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])),
+                Arc::new(StringArray::from(vec![
+                    Some(""),
+                    Some("   "),
+                    Some("the"),
+                    Some("overlength"),
+                    None,
+                    Some("hello"),
+                ])),
+            ],
+        )
+        .unwrap();
+        let index = FtsMemIndex::with_params(1, "description".to_string(), params.clone());
+        index.insert(&batch, 0).unwrap();
+
+        assert_eq!(index.doc_count(), 1);
+        let st = index.state.load_full();
+        let tail_snap = st.tail.snapshot();
+        let tokens = vec!["hello".to_string()];
+        let tail_scorer = build_scorer(&st, &tail_snap, &tokens, true);
+        let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)]));
+        assert_eq!(tail_scorer.total_tokens, 1);
+        assert_eq!(tail_scorer.num_docs(), 1);
+        assert_eq!(tail_scorer.num_docs_containing_token("hello"), 1);
+        assert_eq!(
+            tail_scorer.avg_doc_length(),
+            expected_scorer.avg_doc_length()
+        );
+        assert_eq!(
+            tail_scorer.query_weight("hello"),
+            expected_scorer.query_weight("hello")
+        );
+        let tail_results = index.search("hello");
+        assert_eq!(rows(tail_results.clone()), vec![5]);
+        let tail_score = tail_results[0].score;
+        assert!(!index.to_index_builder(0, 6).unwrap().is_empty());
+
+        index.flush();
+        let st = index.state.load_full();
+        assert_eq!(st.partitions.len(), 1);
+        assert_eq!(
+            st.partitions[0]
+                .docs
+                .iter()
+                .map(|(row_id, num_tokens)| (*row_id, *num_tokens))
+                .collect::<Vec<_>>(),
+            vec![(5, 1)]
+        );
+        let frozen_scorer = build_scorer(&st, &st.tail.snapshot(), &tokens, true);
+        assert_eq!(frozen_scorer.total_tokens, 1);
+        assert_eq!(frozen_scorer.num_docs(), 1);
+        assert_eq!(frozen_scorer.num_docs_containing_token("hello"), 1);
+        assert_eq!(
+            frozen_scorer.avg_doc_length(),
+            expected_scorer.avg_doc_length()
+        );
+        assert_eq!(
+            frozen_scorer.query_weight("hello"),
+            expected_scorer.query_weight("hello")
+        );
+        let frozen_results = index.search("hello");
+        assert_eq!(rows(frozen_results.clone()), vec![5]);
+        assert!((frozen_results[0].score - tail_score).abs() < f32::EPSILON);
+
+        let all_zero_batch = batch.slice(0, 5);
+        let all_zero_index = FtsMemIndex::with_params(1, "description".to_string(), params);
+        all_zero_index.insert(&all_zero_batch, 0).unwrap();
+        assert!(all_zero_index.is_empty());
+        assert_eq!(all_zero_index.doc_count(), 0);
+        assert!(all_zero_index.to_index_builder(0, 5).unwrap().is_empty());
+        all_zero_index.flush();
+        assert!(all_zero_index.state.load().partitions.is_empty());
+    }
+
     fn create_phrase_test_batch(schema: &ArrowSchema) -> RecordBatch {
         RecordBatch::try_new(
             Arc::new(schema.clone()),
@@ -4666,6 +5199,18 @@ mod tests {
         assert!(builder.id() > 0 || builder.id() == 42);
     }
 
+    #[test]
+    fn test_to_index_builder_supports_block_size_256() {
+        let schema = create_test_schema();
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let index = FtsMemIndex::try_with_params(1, "description".to_string(), params).unwrap();
+        let batch = create_test_batch(&schema);
+        index.insert(&batch, 0).unwrap();
+
+        let builder = index.to_index_builder(42, 3).unwrap();
+        assert_eq!(builder.id(), 42);
+    }
+
     #[test]
     fn test_unsupported_column_type_errors() {
         let schema = Arc::new(ArrowSchema::new(vec![
diff --git a/rust/lance/src/dataset/mem_wal/manifest.rs b/rust/lance/src/dataset/mem_wal/manifest.rs
index 9c2a3aa2163..acfcbcc3a7c 100644
--- a/rust/lance/src/dataset/mem_wal/manifest.rs
+++ b/rust/lance/src/dataset/mem_wal/manifest.rs
@@ -146,7 +146,7 @@ impl ShardManifestStore {
             replay_after_wal_entry_position: 0,
             wal_entry_position_last_seen: 0,
             current_generation: 1,
-            flushed_generations: vec![],
+            sstables: vec![],
             status: ShardStatus::Active,
         };
 
@@ -462,7 +462,7 @@ impl ShardManifestStore {
                     replay_after_wal_entry_position: 0,
                     wal_entry_position_last_seen: 0,
                     current_generation: 1,
-                    flushed_generations: vec![],
+                    sstables: vec![],
                     status: ShardStatus::Active,
                 }
             };
@@ -619,7 +619,7 @@ mod tests {
             replay_after_wal_entry_position: 0,
             wal_entry_position_last_seen: 0,
             current_generation: 1,
-            flushed_generations: vec![],
+            sstables: vec![],
             status: ShardStatus::Active,
         }
     }
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index 77611fd7c47..102271bfd23 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -7,7 +7,6 @@ pub mod batch_store;
 pub mod flush;
 pub mod scanner;
 
-use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -79,13 +78,6 @@ pub struct MemTable {
     /// Generation number (incremented on flush).
     generation: u64,
 
-    /// WAL batch mapping: batch_position -> (wal_entry_position, position within WAL entry).
-    wal_batch_mapping: HashMap<usize, (u64, usize)>,
-    /// Last WAL entry position that has been flushed.
-    last_flushed_wal_entry_position: u64,
-    /// Set of batch IDs that have been flushed to WAL.
-    flushed_batch_positions: HashSet<usize>,
-
     /// Primary key bloom filter for staleness detection.
     pk_bloom_filter: Sbbf,
     /// Primary key field IDs (for bloom filter updates).
@@ -187,6 +179,29 @@ impl MemTable {
         pk_field_ids: Vec<i32>,
         cache_config: CacheConfig,
         batch_capacity: usize,
+    ) -> Result<Self> {
+        Self::with_capacity_at(
+            schema,
+            generation,
+            pk_field_ids,
+            cache_config,
+            batch_capacity,
+            0,
+        )
+    }
+
+    /// Create a memtable whose batch 0 sits at `global_offset` in the writer's
+    /// batch sequence. Every memtable after the writer's first is rotated in by
+    /// `freeze_memtable`, which stamps the outgoing memtable's `global_end()`
+    /// here so writer-global cursors stay mappable onto local batch positions.
+    #[allow(clippy::too_many_arguments)]
+    pub fn with_capacity_at(
+        schema: Arc<ArrowSchema>,
+        generation: u64,
+        pk_field_ids: Vec<i32>,
+        cache_config: CacheConfig,
+        batch_capacity: usize,
+        global_offset: usize,
     ) -> Result<Self> {
         let lance_schema = Schema::try_from(schema.as_ref())?;
 
@@ -205,7 +220,7 @@ impl MemTable {
         let dataset_uri = format!("memory://{}", Uuid::new_v4());
 
         // Create lock-free batch store
-        let batch_store = Arc::new(BatchStore::with_capacity(batch_capacity));
+        let batch_store = Arc::new(BatchStore::with_capacity_at(batch_capacity, global_offset));
 
         // Create memtable_flush_completion cell immediately so backpressure can
         // wait on it even before the memtable is frozen. Every memtable will
@@ -220,9 +235,6 @@ impl MemTable {
             cache_config,
             cached_dataset: RwLock::new(None),
             generation,
-            wal_batch_mapping: HashMap::new(),
-            last_flushed_wal_entry_position: 0,
-            flushed_batch_positions: HashSet::new(),
             pk_bloom_filter,
             pk_field_ids,
             // Initialize with an empty IndexStore so the visibility cursor has
@@ -481,53 +493,41 @@ impl MemTable {
         self.batch_store.is_full() || self.batch_store.estimated_bytes() >= max_bytes
     }
 
-    /// Get batches visible up to a specific batch position (inclusive).
+    /// Get the batches in the visible prefix.
     ///
-    /// A batch at position `i` is visible if `i <= max_visible_batch_position`.
+    /// A batch at position `i` is visible if `i < visible_count`.
     ///
     /// # Arguments
     ///
-    /// * `max_visible_batch_position` - The maximum batch position to include (inclusive)
+    /// * `visible_count` - Exclusive count of batch positions to include
     ///
     /// # Returns
     ///
     /// Vector of visible batches.
-    pub async fn get_visible_batches(&self, max_visible_batch_position: usize) -> Vec<RecordBatch> {
-        self.batch_store
-            .visible_record_batches(max_visible_batch_position)
+    pub async fn get_visible_batches(&self, visible_count: usize) -> Vec<RecordBatch> {
+        self.batch_store.visible_record_batches(visible_count)
     }
 
-    /// Get batch positions visible up to a specific batch position (inclusive).
+    /// Get the batch positions in the visible prefix.
     ///
     /// This is useful for filtering index results by visibility.
-    pub async fn get_max_visible_batch_positions(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Vec<usize> {
-        self.batch_store
-            .max_visible_batch_positions(max_visible_batch_position)
+    pub async fn get_visible_batch_positions(&self, visible_count: usize) -> Vec<usize> {
+        self.batch_store.visible_batch_positions(visible_count)
     }
 
     /// Check if a specific batch is visible at a given visibility position.
     ///
     /// Returns true if the batch is visible, false if not visible or doesn't exist.
-    pub async fn is_batch_visible(
-        &self,
-        batch_position: usize,
-        max_visible_batch_position: usize,
-    ) -> bool {
+    pub async fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool {
         self.batch_store
-            .is_batch_visible(batch_position, max_visible_batch_position)
+            .is_batch_visible(batch_position, visible_count)
     }
 
     /// Scan batches visible up to a specific batch position.
     ///
     /// This combines `get_visible_batches` with the scan interface.
-    pub async fn scan_batches_at_position(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Result<Vec<RecordBatch>> {
-        Ok(self.get_visible_batches(max_visible_batch_position).await)
+    pub async fn scan_batches_at_position(&self, visible_count: usize) -> Result<Vec<RecordBatch>> {
+        Ok(self.get_visible_batches(visible_count).await)
     }
 
     /// Update the bloom filter with primary keys from a batch.
@@ -561,30 +561,6 @@ impl MemTable {
         Ok(())
     }
 
-    /// Mark batches as flushed to WAL.
-    ///
-    /// Updates the WAL batch mapping for use during MemTable flush.
-    /// Also updates the batch_store's watermark to the highest flushed batch_position.
-    pub fn mark_wal_flushed(
-        &mut self,
-        batch_positions: &[usize],
-        wal_entry_position: u64,
-        positions: &[usize],
-    ) {
-        for (idx, &batch_position) in batch_positions.iter().enumerate() {
-            self.wal_batch_mapping
-                .insert(batch_position, (wal_entry_position, positions[idx]));
-            self.flushed_batch_positions.insert(batch_position);
-        }
-        self.last_flushed_wal_entry_position = wal_entry_position;
-
-        // Update batch_store watermark to the highest batch_position flushed (inclusive)
-        if let Some(&max_batch_position) = batch_positions.iter().max() {
-            self.batch_store
-                .set_max_flushed_batch_position(max_batch_position);
-        }
-    }
-
     /// Get or create a Dataset for reading.
     ///
     /// Uses caching based on the configured eventual consistency strategy:
@@ -654,7 +630,7 @@ impl MemTable {
     ///
     /// This is used when flushing MemTable to persistent storage to ensure
     /// the flushed data is ordered from newest to oldest. This enables more
-    /// efficient K-way merge during LSM scan because flushed generations
+    /// efficient K-way merge during LSM scan because SSTables
     /// will be pre-sorted in the order needed for deduplication.
     ///
     /// The total number of rows in the MemTable is also returned to allow
@@ -726,16 +702,6 @@ impl MemTable {
         self.batch_store.estimated_bytes() + self.pk_bloom_filter.estimated_memory_size()
     }
 
-    /// Get the WAL batch mapping.
-    pub fn wal_batch_mapping(&self) -> &HashMap<usize, (u64, usize)> {
-        &self.wal_batch_mapping
-    }
-
-    /// Get the last flushed WAL entry position.
-    pub fn last_flushed_wal_entry_position(&self) -> u64 {
-        self.last_flushed_wal_entry_position
-    }
-
     /// Get the bloom filter for serialization.
     pub fn bloom_filter(&self) -> &Sbbf {
         &self.pk_bloom_filter
@@ -757,17 +723,15 @@ impl MemTable {
         self.indexes.take()
     }
 
-    /// Check if all batches have been flushed to WAL.
-    pub fn all_flushed_to_wal(&self) -> bool {
-        self.batch_store.pending_wal_flush_count() == 0
+    /// Whether every committed batch in this memtable is WAL-durable, given the
+    /// writer-global durability cursor. The L0 flush's precondition.
+    pub fn all_flushed_to_wal(&self, durable: usize) -> bool {
+        self.batch_store.pending_wal_flush_count(durable) == 0
     }
 
-    /// Get unflushed batch IDs.
-    pub fn unflushed_batch_positions(&self) -> Vec<usize> {
-        let batch_count = self.batch_count();
-        (0..batch_count)
-            .filter(|id| !self.flushed_batch_positions.contains(id))
-            .collect()
+    /// Writer-global coordinate one past this memtable's last committed batch.
+    pub fn global_end(&self) -> usize {
+        self.batch_store.global_end()
     }
 
     /// Get cache configuration.
@@ -785,18 +749,13 @@ impl MemTable {
         self.batch_store.remaining_capacity()
     }
 
-    /// Check if batch store is full.
-    pub fn is_batch_store_full(&self) -> bool {
-        self.batch_store.is_full()
-    }
-
     /// Create a scanner for querying this MemTable.
     ///
     /// # Arguments
     ///
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     ///
-    /// The scanner captures the current `max_visible_batch_position` from the
+    /// The scanner captures the current `visible_count` from the
     /// `IndexStore` at construction time to ensure consistent visibility.
     ///
     /// # Panics
@@ -931,29 +890,11 @@ mod tests {
         assert_eq!(total_rows, 15);
     }
 
+    /// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`):
+    /// false while any committed batch is still un-appended, true once the
+    /// durability watermark covers every one of them.
     #[tokio::test]
-    async fn test_memtable_wal_mapping() {
-        let schema = create_test_schema();
-        let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
-
-        let batch_position = memtable
-            .insert(create_test_batch(&schema, 10))
-            .await
-            .unwrap();
-        assert!(!memtable.all_flushed_to_wal());
-
-        memtable.mark_wal_flushed(&[batch_position], 5, &[0]);
-
-        assert!(memtable.all_flushed_to_wal());
-        assert_eq!(
-            memtable.wal_batch_mapping().get(&batch_position),
-            Some(&(5, 0))
-        );
-        assert_eq!(memtable.last_flushed_wal_entry_position(), 5);
-    }
-
-    #[tokio::test]
-    async fn test_memtable_unflushed_batches() {
+    async fn test_all_flushed_to_wal_tracks_the_durability_watermark() {
         let schema = create_test_schema();
         let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
 
@@ -965,12 +906,16 @@ mod tests {
             .insert(create_test_batch(&schema, 5))
             .await
             .unwrap();
+        assert!(!memtable.all_flushed_to_wal(0), "nothing is durable yet");
 
-        assert_eq!(memtable.unflushed_batch_positions(), vec![batch1, batch2]);
-
-        memtable.mark_wal_flushed(&[batch1], 1, &[0]);
+        let durable = batch1 + 1;
+        assert!(
+            !memtable.all_flushed_to_wal(durable),
+            "batch2 is still waiting on its WAL append"
+        );
 
-        assert_eq!(memtable.unflushed_batch_positions(), vec![batch2]);
+        let durable = batch2 + 1;
+        assert!(memtable.all_flushed_to_wal(durable));
     }
 
     #[tokio::test]
@@ -992,23 +937,21 @@ mod tests {
             .await
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 are visible
-        let visible = memtable.get_visible_batches(1).await;
+        // A count of N exposes the prefix [0, N).
+        let visible = memtable.get_visible_batches(2).await;
         assert_eq!(visible.len(), 2);
         let total_rows: usize = visible.iter().map(|b| b.num_rows()).sum();
         assert_eq!(total_rows, 15); // 10 + 5
 
-        // max_visible_batch_position=2 means all batches are visible
-        let visible = memtable.get_visible_batches(2).await;
+        let visible = memtable.get_visible_batches(3).await;
         assert_eq!(visible.len(), 3);
 
-        // max_visible_batch_position=0 means only position 0 is visible
-        let visible = memtable.get_visible_batches(0).await;
-        assert_eq!(visible.len(), 1);
+        // A count of 0 exposes nothing — not "batch 0".
+        assert!(memtable.get_visible_batches(0).await.is_empty());
     }
 
     #[tokio::test]
-    async fn test_memtable_get_max_visible_batch_positions() {
+    async fn test_memtable_get_visible_batch_positions() {
         let schema = create_test_schema();
         let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
 
@@ -1026,17 +969,15 @@ mod tests {
             .await
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 visible
-        let visible_ids = memtable.get_max_visible_batch_positions(1).await;
+        // A count of N exposes the prefix [0, N).
+        let visible_ids = memtable.get_visible_batch_positions(2).await;
         assert_eq!(visible_ids, vec![0, 1]);
 
-        // max_visible_batch_position=2 means all positions visible
-        let visible_ids = memtable.get_max_visible_batch_positions(2).await;
+        let visible_ids = memtable.get_visible_batch_positions(3).await;
         assert_eq!(visible_ids, vec![0, 1, 2]);
 
-        // max_visible_batch_position=0 means only position 0 visible
-        let visible_ids = memtable.get_max_visible_batch_positions(0).await;
-        assert_eq!(visible_ids, vec![0]);
+        // A count of 0 exposes nothing.
+        assert!(memtable.get_visible_batch_positions(0).await.is_empty());
     }
 
     #[tokio::test]
@@ -1057,14 +998,14 @@ mod tests {
             .await
             .unwrap(); // position 2
 
-        // batch_position 0 is visible when max_visible_batch_position >= 0
-        assert!(memtable.is_batch_visible(0, 0).await);
+        // A count of 0 means nothing is visible, batch 0 included.
+        assert!(!memtable.is_batch_visible(0, 0).await);
+
+        // Batch i is visible once the count exceeds i.
         assert!(memtable.is_batch_visible(0, 1).await);
         assert!(memtable.is_batch_visible(0, 2).await);
-
-        // batch_position 2 is only visible when max_visible_batch_position >= 2
         assert!(!memtable.is_batch_visible(2, 1).await);
-        assert!(memtable.is_batch_visible(2, 2).await);
+        assert!(!memtable.is_batch_visible(2, 2).await);
         assert!(memtable.is_batch_visible(2, 3).await);
 
         // Non-existent batch
@@ -1085,12 +1026,21 @@ mod tests {
             .await
             .unwrap(); // position 1
 
-        let batches = memtable.scan_batches_at_position(0).await.unwrap();
+        let batches = memtable.scan_batches_at_position(1).await.unwrap();
         assert_eq!(batches.len(), 1);
         assert_eq!(batches[0].num_rows(), 10);
 
-        let batches = memtable.scan_batches_at_position(1).await.unwrap();
+        let batches = memtable.scan_batches_at_position(2).await.unwrap();
         assert_eq!(batches.len(), 2);
+
+        // Nothing indexed yet => nothing scannable.
+        assert!(
+            memtable
+                .scan_batches_at_position(0)
+                .await
+                .unwrap()
+                .is_empty()
+        );
     }
 
     #[tokio::test]
@@ -1101,7 +1051,7 @@ mod tests {
 
         assert_eq!(memtable.batch_capacity(), 3);
         assert_eq!(memtable.remaining_batch_capacity(), 3);
-        assert!(!memtable.is_batch_store_full());
+        assert!(!memtable.batch_store().is_full());
 
         // Fill up the store
         memtable
@@ -1117,7 +1067,7 @@ mod tests {
             .await
             .unwrap();
 
-        assert!(memtable.is_batch_store_full());
+        assert!(memtable.batch_store().is_full());
         assert_eq!(memtable.remaining_batch_capacity(), 0);
 
         // Next insert should fail
diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
index 054d9b1630e..f16f9bea1fd 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
@@ -43,7 +43,9 @@ use std::cell::UnsafeCell;
 use std::mem::MaybeUninit;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
+use arrow::array::ArrayData;
 use arrow_array::RecordBatch;
+use arrow_schema::DataType;
 
 /// A batch stored in the lock-free store.
 #[derive(Clone)]
@@ -75,14 +77,56 @@ impl StoredBatch {
     }
 
     /// Estimate the memory size of a RecordBatch.
+    ///
+    /// Sums each column's slice-aware buffer size (see
+    /// [`Self::estimate_array_size`]) plus the struct overhead, so a column that
+    /// is a zero-copy slice of a larger parent contributes only its own window
+    /// rather than the whole shared buffer.
     fn estimate_batch_size(batch: &RecordBatch) -> usize {
         batch
             .columns()
             .iter()
-            .map(|col| col.get_array_memory_size())
+            .map(|col| Self::estimate_array_size(&col.to_data()))
             .sum::<usize>()
             + std::mem::size_of::<RecordBatch>()
     }
+
+    /// Slice-aware buffer size of a single array.
+    ///
+    /// [`ArrayData::get_slice_memory_size`] reports each buffer's own window
+    /// (not the whole shared buffer), but omits the variadic data buffers of
+    /// `Utf8View`/`BinaryView` (values > 12 bytes) while still returning `Ok`, so
+    /// [`Self::view_data_buffers_size`] adds them. Those buffers are shared across
+    /// zero-copy slices and are counted at full capacity for each slice — an
+    /// over-count in the safe direction.
+    fn estimate_array_size(data: &ArrayData) -> usize {
+        match data.get_slice_memory_size() {
+            Ok(size) => size + Self::view_data_buffers_size(data),
+            // Fall back to the full-buffer sum for layouts the slice-aware call
+            // cannot handle.
+            Err(_) => data.get_array_memory_size(),
+        }
+    }
+
+    /// Capacity of the variadic `Utf8View`/`BinaryView` data buffers that
+    /// [`ArrayData::get_slice_memory_size`] omits, summed recursively over children.
+    fn view_data_buffers_size(data: &ArrayData) -> usize {
+        let mut size = 0;
+        if matches!(data.data_type(), DataType::Utf8View | DataType::BinaryView) {
+            // buffers()[0] is the 16-byte view array that get_slice_memory_size
+            // already counts; [1..] are the data buffers it skips.
+            size += data
+                .buffers()
+                .iter()
+                .skip(1)
+                .map(|b| b.capacity())
+                .sum::<usize>();
+        }
+        for child in data.child_data() {
+            size += Self::view_data_buffers_size(child);
+        }
+        size
+    }
 }
 
 /// Snapshot of the active batches that have not yet been flushed to WAL.
@@ -151,10 +195,14 @@ pub struct BatchStore {
     /// Estimated size in bytes (for flush threshold).
     estimated_bytes: AtomicUsize,
 
-    /// WAL flush watermark: the last batch ID that has been flushed to WAL (inclusive).
-    /// Uses usize::MAX as sentinel for "nothing flushed yet".
-    /// This is per-memtable tracking, not global.
-    max_flushed_batch_position: AtomicUsize,
+    /// Writer-global coordinate of this store's batch 0.
+    ///
+    /// A *coordinate*, not a cursor: stamped once at construction and never
+    /// moved. `global_position = global_offset + local_position`. Batch
+    /// positions restart at 0 in every memtable, so this is the only thing that
+    /// lets a writer-global cursor (the WAL durability count) be mapped onto a
+    /// particular store.
+    global_offset: usize,
 }
 
 // SAFETY: Safe to share across threads because:
@@ -177,6 +225,13 @@ impl BatchStore {
     ///
     /// Panics if capacity is 0.
     pub fn with_capacity(capacity: usize) -> Self {
+        Self::with_capacity_at(capacity, 0)
+    }
+
+    /// Create a store whose batch 0 sits at `global_offset` in the writer's
+    /// batch sequence. Used by `freeze_memtable` for every memtable after the
+    /// first; the first starts at 0.
+    pub fn with_capacity_at(capacity: usize, global_offset: usize) -> Self {
         assert!(capacity > 0, "capacity must be > 0");
 
         // Allocate uninitialized storage
@@ -191,7 +246,7 @@ impl BatchStore {
             capacity,
             total_rows: AtomicUsize::new(0),
             estimated_bytes: AtomicUsize::new(0),
-            max_flushed_batch_position: AtomicUsize::new(usize::MAX), // Nothing flushed yet
+            global_offset,
         }
     }
 
@@ -393,74 +448,54 @@ impl BatchStore {
     // WAL Flush Tracking API
     // =========================================================================
 
-    /// Get the WAL flush watermark (the last batch ID that was flushed, inclusive).
-    /// Returns None if nothing has been flushed yet.
+    /// Writer-global coordinate one past this store's last committed batch.
     #[inline]
-    pub fn max_flushed_batch_position(&self) -> Option<usize> {
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        if watermark == usize::MAX {
-            None
-        } else {
-            Some(watermark)
-        }
+    pub fn global_end(&self) -> usize {
+        self.global_offset + self.committed_len.load(Ordering::Acquire)
     }
 
-    /// Update the WAL flush watermark after successful WAL flush.
-    ///
-    /// # Arguments
-    ///
-    /// * `batch_position` - The last batch ID that was flushed (inclusive)
+    /// This store's writer-global coordinate for batch 0.
     #[inline]
-    pub fn set_max_flushed_batch_position(&self, batch_position: usize) {
-        debug_assert!(
-            batch_position != usize::MAX,
-            "batch_position cannot be usize::MAX (reserved as sentinel)"
-        );
-        self.max_flushed_batch_position
-            .store(batch_position, Ordering::Release);
+    pub fn global_offset(&self) -> usize {
+        self.global_offset
     }
 
-    /// Get the number of batches pending WAL flush.
+    /// The local exclusive end of this store covered by a writer-global cursor.
+    ///
+    /// Saturating in both directions, and both directions are reachable in
+    /// normal operation: a cursor *below* this store's offset means "nothing
+    /// here yet" (the store was rotated in after the cursor last advanced —
+    /// the ordinary state of a fresh memtable), and a cursor beyond its end
+    /// clamps to what is committed.
+    ///
+    /// This is the **only** place the global-to-local subtraction is written.
+    /// Open-coding it underflows on every memtable rotation, which in release
+    /// wraps to a huge end and makes the whole new memtable instantly visible.
     #[inline]
-    pub fn pending_wal_flush_count(&self) -> usize {
-        let committed = self.committed_len.load(Ordering::Acquire);
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        if watermark == usize::MAX {
-            // Nothing flushed yet, all committed batches are pending
-            committed
-        } else {
-            // Batches [0, watermark] are flushed, so pending = committed - (watermark + 1)
-            committed.saturating_sub(watermark + 1)
-        }
+    pub fn local_end(&self, global_cursor: usize) -> usize {
+        global_cursor
+            .saturating_sub(self.global_offset)
+            .min(self.committed_len.load(Ordering::Acquire))
     }
 
-    /// Check if all committed batches have been WAL-flushed.
+    /// Batches in this store still waiting on their WAL append.
     #[inline]
-    pub fn is_wal_flush_complete(&self) -> bool {
-        self.pending_wal_flush_count() == 0
+    pub fn pending_wal_flush_count(&self, durable: usize) -> usize {
+        self.committed_len.load(Ordering::Acquire) - self.local_end(durable)
     }
 
-    /// Get the range of batch IDs pending WAL flush: [start, end).
-    /// Returns None if nothing pending.
+    /// Local range `[start, end)` of batches still waiting on their WAL append,
+    /// or `None` when the store is fully durable.
     #[inline]
-    pub fn pending_wal_flush_range(&self) -> Option<(usize, usize)> {
-        let committed = self.committed_len.load(Ordering::Acquire);
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        let start = if watermark == usize::MAX {
-            0
-        } else {
-            watermark + 1
-        };
-        if committed > start {
-            Some((start, committed))
-        } else {
-            None
-        }
+    pub fn pending_wal_flush_range(&self, durable: usize) -> Option<(usize, usize)> {
+        let start = self.local_end(durable);
+        let end = self.committed_len.load(Ordering::Acquire);
+        (end > start).then_some((start, end))
     }
 
     /// Get a point-in-time summary of batches pending WAL flush.
-    pub fn pending_wal_flush_stats(&self) -> PendingWalFlushStats {
-        let Some((start, end)) = self.pending_wal_flush_range() else {
+    pub fn pending_wal_flush_stats(&self, durable: usize) -> PendingWalFlushStats {
+        let Some((start, end)) = self.pending_wal_flush_range(durable) else {
             return PendingWalFlushStats::default();
         };
 
@@ -599,66 +634,53 @@ impl BatchStore {
     // Visibility API
     // =========================================================================
 
-    /// Get batches visible up to a specific batch position (inclusive).
+    /// Batches in the visible prefix `[0, visible_count)`.
     ///
-    /// A batch at position `i` is visible if `i <= max_visible_batch_position`.
-    pub fn visible_batches(&self, max_visible_batch_position: usize) -> Vec<&StoredBatch> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        let end = (max_visible_batch_position + 1).min(len);
+    /// `visible_count` is an **exclusive count**, not an inclusive position: 0
+    /// means nothing is visible. As an inclusive position, 0 meant *both*
+    /// "nothing visible" and "batch 0 is visible", so a batch that was committed
+    /// to the store but not yet indexed or WAL-durable was readable for a full
+    /// PUT round-trip. The count makes that off-by-one inexpressible.
+    pub fn visible_batches(&self, visible_count: usize) -> Vec<&StoredBatch> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
         (0..end).filter_map(|i| self.get(i)).collect()
     }
 
-    /// Get batch positions visible up to a specific batch position (inclusive).
-    pub fn max_visible_batch_positions(&self, max_visible_batch_position: usize) -> Vec<usize> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        let end = (max_visible_batch_position + 1).min(len);
+    /// Positions of the batches in the visible prefix.
+    pub fn visible_batch_positions(&self, visible_count: usize) -> Vec<usize> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
         (0..end).collect()
     }
 
-    /// The inclusive maximum visible *row* position at `max_visible_batch_position`,
-    /// or `None` when no rows are visible. The visible batches are the committed
-    /// prefix `[0, last_visible_idx]`; each batch carries its cumulative
-    /// `row_offset`, so this is the end of the last visible batch minus one.
-    /// Used to bound MVCC seeks against the maintained PK-position index.
-    pub fn max_visible_row(&self, max_visible_batch_position: usize) -> Option<u64> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        if len == 0 {
-            return None;
-        }
-        let last_visible_idx = max_visible_batch_position.min(len - 1);
-        let last = self.get(last_visible_idx)?;
+    /// The inclusive maximum visible *row* position, or `None` when no rows are
+    /// visible. Each batch carries its cumulative `row_offset`, so this is the
+    /// end of the last visible batch minus one. Bounds MVCC seeks against the
+    /// maintained PK-position index.
+    pub fn max_visible_row(&self, visible_count: usize) -> Option<u64> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
+        let last = self.get(end.checked_sub(1)?)?;
         let visible_end = last.row_offset + last.num_rows as u64; // exclusive
         visible_end.checked_sub(1)
     }
 
-    /// Check if a specific batch is visible at a given visibility position.
+    /// Whether a batch falls inside the visible prefix.
     #[inline]
-    pub fn is_batch_visible(
-        &self,
-        batch_position: usize,
-        max_visible_batch_position: usize,
-    ) -> bool {
+    pub fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool {
         let len = self.committed_len.load(Ordering::Acquire);
-        batch_position < len && batch_position <= max_visible_batch_position
+        batch_position < len && batch_position < visible_count
     }
 
-    /// Get visible RecordBatches (clones the data).
-    pub fn visible_record_batches(&self, max_visible_batch_position: usize) -> Vec<RecordBatch> {
-        self.visible_batches(max_visible_batch_position)
+    /// Visible RecordBatches (clones the data).
+    pub fn visible_record_batches(&self, visible_count: usize) -> Vec<RecordBatch> {
+        self.visible_batches(visible_count)
             .into_iter()
             .map(|b| b.data.clone())
             .collect()
     }
 
-    /// Get visible RecordBatches with their row offsets.
-    ///
-    /// Returns tuples of (batch, row_offset) for each visible batch.
-    /// The row_offset is the starting row position for that batch.
-    pub fn visible_batches_with_offsets(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Vec<(RecordBatch, u64)> {
-        self.visible_batches(max_visible_batch_position)
+    /// Visible RecordBatches paired with the row position each one starts at.
+    pub fn visible_batches_with_offsets(&self, visible_count: usize) -> Vec<(RecordBatch, u64)> {
+        self.visible_batches(visible_count)
             .into_iter()
             .map(|b| (b.data.clone(), b.row_offset))
             .collect()
@@ -891,17 +913,52 @@ mod tests {
         store.append(create_test_batch(10)).unwrap(); // position 3
         store.append(create_test_batch(10)).unwrap(); // position 4
 
-        // max_visible_batch_position=2 means positions 0, 1, 2 are visible
-        let visible = store.max_visible_batch_positions(2);
-        assert_eq!(visible, vec![0, 1, 2]);
+        // A count of N exposes the prefix [0, N).
+        assert_eq!(store.visible_batch_positions(3), vec![0, 1, 2]);
+        assert_eq!(store.visible_batch_positions(5), vec![0, 1, 2, 3, 4]);
+
+        // A count of 0 exposes nothing. Under the old inclusive cursor this
+        // case was indistinguishable from "batch 0 is visible", so every
+        // memtable leaked its first batch before it was indexed or durable.
+        assert!(store.visible_batch_positions(0).is_empty());
+
+        // Beyond the committed range, clamp.
+        assert_eq!(store.visible_batch_positions(99), vec![0, 1, 2, 3, 4]);
+    }
+
+    /// The zero of the visibility cursor must be unambiguous.
+    ///
+    /// `BatchStore::append` publishes `committed_len` on the put path, under the
+    /// state lock, *before* the WAL flush that indexes the batch is even
+    /// triggered — and that flush is a ~100ms S3 PUT on another task. So batch 0
+    /// sits committed and readable for a full round-trip before it is indexed or
+    /// durable. As an inclusive position, a cursor of 0 meant both "nothing is
+    /// visible" and "batch 0 is visible", so every read arm backed by the batch
+    /// store served that batch while the index-backed arms did not — the tiers
+    /// actively disagreed. An exclusive count makes the state inexpressible.
+    #[test]
+    fn test_zero_cursor_hides_the_committed_but_unindexed_prefix() {
+        let store = BatchStore::with_capacity(4);
+        store.append(create_test_batch(10)).unwrap();
+        store.append(create_test_batch(10)).unwrap();
 
-        // max_visible_batch_position=4 means all visible
-        let visible = store.max_visible_batch_positions(4);
-        assert_eq!(visible, vec![0, 1, 2, 3, 4]);
+        // Committed, but nothing indexed yet: every visibility query must agree
+        // that there is nothing to read.
+        assert!(store.visible_batches(0).is_empty());
+        assert!(store.visible_batch_positions(0).is_empty());
+        assert!(store.visible_record_batches(0).is_empty());
+        assert!(store.visible_batches_with_offsets(0).is_empty());
+        assert!(!store.is_batch_visible(0, 0));
+        assert_eq!(store.max_visible_row(0), None);
+
+        // The batches are there — they are simply not yet published.
+        assert_eq!(store.len(), 2);
 
-        // max_visible_batch_position=0 means only position 0 visible
-        let visible = store.max_visible_batch_positions(0);
-        assert_eq!(visible, vec![0]);
+        // Indexing batch 0 publishes exactly batch 0.
+        assert_eq!(store.visible_batches(1).len(), 1);
+        assert!(store.is_batch_visible(0, 1));
+        assert!(!store.is_batch_visible(1, 1));
+        assert_eq!(store.max_visible_row(1), Some(9));
     }
 
     #[test]
@@ -912,14 +969,17 @@ mod tests {
         store.append(create_test_batch(10)).unwrap(); // position 1
         store.append(create_test_batch(10)).unwrap(); // position 2
 
-        // Batch at position 0 is visible when max_visible_batch_position >= 0
-        assert!(store.is_batch_visible(0, 0));
+        // A count of 0 means *nothing* is visible — including batch 0. As an
+        // inclusive position this case was indistinguishable from "batch 0 is
+        // visible", so a batch that was committed to the store but not yet
+        // indexed or WAL-durable was readable for a full PUT round-trip.
+        assert!(!store.is_batch_visible(0, 0));
+
+        // Batch i is visible once the count exceeds i.
         assert!(store.is_batch_visible(0, 1));
         assert!(store.is_batch_visible(0, 2));
-
-        // Batch at position 2 is only visible when max_visible_batch_position >= 2
         assert!(!store.is_batch_visible(2, 1));
-        assert!(store.is_batch_visible(2, 2));
+        assert!(!store.is_batch_visible(2, 2));
         assert!(store.is_batch_visible(2, 3));
 
         // Batch 3 doesn't exist
@@ -928,7 +988,7 @@ mod tests {
 
     #[test]
     fn test_max_visible_row() {
-        // (1) Empty store: no rows are visible at any position.
+        // (1) Empty store: no rows are visible at any count.
         let store = BatchStore::with_capacity(10);
         assert_eq!(store.max_visible_row(0), None);
         assert_eq!(store.max_visible_row(100), None);
@@ -938,23 +998,24 @@ mod tests {
         store.append(create_test_batch(20)).unwrap(); // position 1
         store.append(create_test_batch(30)).unwrap(); // position 2
 
-        // (2) A position within range yields the inclusive end of that prefix.
-        assert_eq!(store.max_visible_row(0), Some(9)); // batch 0: 0..10
-        assert_eq!(store.max_visible_row(1), Some(29)); // batch 1: 10..30
-        assert_eq!(store.max_visible_row(2), Some(59)); // batch 2: 30..60
+        // (2) A count of 0 means nothing is visible — not "batch 0 is visible".
+        assert_eq!(store.max_visible_row(0), None);
+
+        // (3) A count of N yields the inclusive last row of the prefix [0, N).
+        assert_eq!(store.max_visible_row(1), Some(9)); // batch 0: 0..10
+        assert_eq!(store.max_visible_row(2), Some(29)); // + batch 1: 10..30
+        assert_eq!(store.max_visible_row(3), Some(59)); // + batch 2: 30..60
 
-        // (3) A position beyond the committed range clamps to the last batch,
-        // i.e. the inclusive max over all rows.
+        // (4) A count beyond the committed range clamps to the last batch.
         assert_eq!(store.max_visible_row(100), Some(59));
 
-        // (4) An empty leading batch contributes no rows: at its own position
-        // the inclusive end underflows to None, while a later non-empty batch
-        // is reported correctly.
+        // (5) An empty leading batch contributes no rows, so a prefix covering
+        // only it still yields None, while a later non-empty batch is reported.
         let store = BatchStore::with_capacity(10);
         store.append(create_test_batch(0)).unwrap(); // position 0: rows [0,0)
         store.append(create_test_batch(5)).unwrap(); // position 1: rows [0,5)
-        assert_eq!(store.max_visible_row(0), None); // empty prefix → no rows
-        assert_eq!(store.max_visible_row(1), Some(4)); // through batch 1
+        assert_eq!(store.max_visible_row(1), None); // empty prefix → no rows
+        assert_eq!(store.max_visible_row(2), Some(4)); // through batch 1
     }
 
     #[test]
@@ -972,6 +1033,122 @@ mod tests {
         assert_eq!(cap, 16); // minimum
     }
 
+    #[test]
+    fn test_estimated_size_is_slice_aware() {
+        // A batch that is a zero-copy slice of a larger parent must contribute
+        // only its own window to the estimate, not the whole shared buffer.
+        // `get_array_memory_size` counts every buffer's full capacity regardless
+        // of offset/length, so N slices tiling one parent each report the
+        // parent's size and inflate the memtable estimate ~N×, tripping the
+        // flush threshold far below the configured size.
+        let chunk = 1_000;
+        let num_slices = 100;
+        let parent = create_test_batch(chunk * num_slices);
+
+        // One window vs an equivalently-sized owned batch should track each
+        // other; the buggy per-slice estimate would be ~num_slices× larger.
+        let slice_est = StoredBatch::estimate_batch_size(&parent.slice(0, chunk));
+        let owned_est = StoredBatch::estimate_batch_size(&create_test_batch(chunk));
+        assert!(
+            slice_est <= owned_est * 2,
+            "slice estimate {slice_est} should track its own window (~{owned_est}), not the parent"
+        );
+
+        // End-to-end: tiling the parent with zero-copy slices must not multiply
+        // the store's running estimate. Track what the old full-buffer behavior
+        // would have summed to for contrast.
+        let store = BatchStore::with_capacity(num_slices);
+        let mut over_counting_sum = 0usize;
+        for k in 0..num_slices {
+            let s = parent.slice(k * chunk, chunk);
+            over_counting_sum += s
+                .columns()
+                .iter()
+                .map(|col| col.get_array_memory_size())
+                .sum::<usize>()
+                + std::mem::size_of::<RecordBatch>();
+            store.append(s).unwrap();
+        }
+
+        // Two non-nullable Int32 columns → exactly 4 bytes/row/col of payload.
+        let payload_bytes = num_slices * chunk * 2 * std::mem::size_of::<i32>();
+        let estimated = store.estimated_bytes();
+        assert!(
+            estimated >= payload_bytes,
+            "estimate {estimated} should cover the actual payload {payload_bytes}"
+        );
+        // The old behavior over-counts by ~num_slices×; the fix must be far
+        // below it (generous 10× margin against struct/alignment overhead).
+        assert!(
+            estimated * 10 < over_counting_sum,
+            "estimate {estimated} should be far below the over-counting sum {over_counting_sum}"
+        );
+    }
+
+    #[test]
+    fn test_estimated_size_counts_view_data_buffers() {
+        // Long Utf8View/BinaryView values live in variadic data buffers that
+        // `get_slice_memory_size` ignores (returning ~16 * rows). The estimate
+        // must include them, both for a top-level view column and for a view
+        // array nested in a container, which is only reached via child_data
+        // recursion.
+        use arrow_array::{Array, ArrayRef, StringViewArray, StructArray};
+
+        let num_rows = 1_000;
+        // Each value exceeds the 12-byte inline limit, so it spills to a data buffer.
+        let long_value = "x".repeat(64);
+        let payload_bytes = num_rows * long_value.len();
+        // What the slice-aware call alone reports: just the 16-byte view entries.
+        let view_entries_only = num_rows * 16;
+
+        let make_views = || {
+            StringViewArray::from(
+                (0..num_rows)
+                    .map(|_| Some(long_value.as_str()))
+                    .collect::<Vec<_>>(),
+            )
+        };
+        let assert_covers = |batch: &RecordBatch| {
+            let estimated = StoredBatch::estimate_batch_size(batch);
+            assert!(
+                estimated >= payload_bytes,
+                "estimate {estimated} should cover the view data-buffer payload {payload_bytes}"
+            );
+            assert!(
+                estimated > view_entries_only * 2,
+                "estimate {estimated} must exceed the ~{view_entries_only}-byte view-entry-only undercount"
+            );
+        };
+
+        // Top-level view column.
+        let flat = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "s",
+                DataType::Utf8View,
+                false,
+            )])),
+            vec![Arc::new(make_views())],
+        )
+        .unwrap();
+        assert_covers(&flat);
+
+        // View nested inside a struct — reachable only through child_data recursion.
+        let nested = StructArray::from(vec![(
+            Arc::new(Field::new("s", DataType::Utf8View, false)),
+            Arc::new(make_views()) as ArrayRef,
+        )]);
+        let nested = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "st",
+                nested.data_type().clone(),
+                false,
+            )])),
+            vec![Arc::new(nested)],
+        )
+        .unwrap();
+        assert_covers(&nested);
+    }
+
     #[test]
     fn test_to_vec() {
         let store = BatchStore::with_capacity(10);
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index 410823c31db..0a5c97ee6c7 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -12,9 +12,9 @@ use lance_core::cache::LanceCache;
 use lance_core::utils::deletion::DeletionVector;
 use lance_core::{Error, Result};
 use lance_index::IndexType;
-use lance_index::mem_wal::{FlushedGeneration, ShardManifest};
+use lance_index::mem_wal::{ShardManifest, SsTable};
 use lance_index::scalar::{IndexStore, ScalarIndexParams};
-use lance_io::object_store::ObjectStore;
+use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_table::format::IndexMetadata;
 use lance_table::io::commit::write_manifest_file_to_path;
 use lance_table::io::deletion::write_deletion_file;
@@ -28,14 +28,16 @@ use uuid::Uuid;
 use super::super::index::MemIndexConfig;
 use super::super::memtable::MemTable;
 use crate::Dataset;
+use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::mem_wal::manifest::ShardManifestStore;
-use crate::dataset::mem_wal::scanner::GenerationWarmer;
+use crate::dataset::mem_wal::scanner::SsTableWarmer;
 use crate::dataset::mem_wal::scanner::exec::{compute_pk_hash, validate_pk_types};
-use crate::dataset::mem_wal::util::{flushed_memtable_path, generate_random_hash};
+use crate::dataset::mem_wal::util::{derived_store_params, generate_random_hash, sstable_path};
+use crate::session::Session;
 
 #[derive(Debug, Clone)]
 pub struct FlushResult {
-    pub generation: FlushedGeneration,
+    pub sstable: SsTable,
     pub rows_flushed: usize,
     pub covered_wal_entry_position: u64,
 }
@@ -71,7 +73,14 @@ pub struct MemTableFlusher {
     manifest_store: Arc<ShardManifestStore>,
     /// When present, each new generation is warmed before it is committed, so
     /// the first query sees zero cold reads. `None` => no warming.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
+    warmer: Option<Arc<dyn SsTableWarmer>>,
+    /// Store params the base dataset was opened with, reused for the flusher's
+    /// own opens + writes. Used verbatim only for the base's own URI; generation
+    /// URIs go through [`derived_store_params`]. `None` opens by URI alone.
+    store_params: Option<ObjectStoreParams>,
+    /// Session for those opens, sharing the base's store registry. `None` opens
+    /// with a fresh session.
+    session: Option<Arc<Session>>,
 }
 
 impl MemTableFlusher {
@@ -89,15 +98,62 @@ impl MemTableFlusher {
             shard_id,
             manifest_store,
             warmer: None,
+            store_params: None,
+            session: None,
         }
     }
 
     /// Attach the warmer fired pre-commit for each new generation.
-    pub fn with_warmer(mut self, warmer: Option<Arc<dyn GenerationWarmer>>) -> Self {
+    pub fn with_warmer(mut self, warmer: Option<Arc<dyn SsTableWarmer>>) -> Self {
         self.warmer = warmer;
         self
     }
 
+    /// Set the store params + session used for derived-URI opens. Injected by
+    /// `mem_wal_writer` from the base `Dataset`.
+    pub fn with_storage_context(
+        mut self,
+        store_params: Option<ObjectStoreParams>,
+        session: Option<Arc<Session>>,
+    ) -> Self {
+        self.store_params = store_params;
+        self.session = session;
+        self
+    }
+
+    /// Open the base table, reusing the injected store params verbatim — they
+    /// were resolved for exactly this URI, so a path-bound `object_store`
+    /// binding still points where it should.
+    async fn open_base(&self) -> Result<Dataset> {
+        self.open_uri(&self.base_uri, self.store_params.clone())
+            .await
+    }
+
+    /// Open an SSTable under `_mem_wal/`. The params must be adapted
+    /// first: a path-bound store binding would redirect the open at the base
+    /// table (see [`derived_store_params`]).
+    async fn open_generation(&self, uri: &str) -> Result<Dataset> {
+        self.open_uri(uri, self.store_params.as_ref().map(derived_store_params))
+            .await
+    }
+
+    /// Open `uri` with the injected session, or by URI alone when nothing was
+    /// injected.
+    async fn open_uri(
+        &self,
+        uri: &str,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Dataset> {
+        let mut builder = DatasetBuilder::from_uri(uri);
+        if let Some(params) = store_params {
+            builder = builder.with_store_params(params);
+        }
+        if let Some(session) = &self.session {
+            builder = builder.with_session(session.clone());
+        }
+        builder.load().await
+    }
+
     /// Warm a just-written generation before it is committed. Best-effort: a
     /// failure is logged and the flush proceeds — warming is never a commit
     /// gate. No-op without a warmer. `uri` must be the resolved reader path
@@ -130,16 +186,16 @@ impl MemTableFlusher {
         }
     }
 
-    /// Storage file version of the shard's base dataset. Flushed generations
+    /// Storage file version of the shard's base dataset. SSTables
     /// (data fragments and index files) are written at this same version so the
-    /// whole shard stays on one format (e.g. a 2.2 base => 2.2 flushed gens).
+    /// whole shard stays on one format (e.g. a 2.2 base => 2.2 SSTables).
     ///
     /// Falls back to [`LanceFileVersion::default`] when no base dataset exists at
     /// `base_uri` (e.g. flusher unit tests that run without a committed base).
     /// In production MemWAL is always initialized on a real dataset, so the base
     /// version is inherited; other open errors are propagated.
     async fn base_storage_version(&self) -> Result<lance_file::version::LanceFileVersion> {
-        match Dataset::open(&self.base_uri).await {
+        match self.open_base().await {
             Ok(dataset) => dataset.manifest().data_storage_format.lance_file_version(),
             Err(Error::DatasetNotFound { .. }) => {
                 Ok(lance_file::version::LanceFileVersion::default())
@@ -161,6 +217,7 @@ impl MemTableFlusher {
         memtable: &MemTable,
         epoch: u64,
         covered_wal_entry_position: u64,
+        durable: usize,
     ) -> Result<FlushResult> {
         self.manifest_store.check_fenced(epoch).await?;
 
@@ -168,7 +225,7 @@ impl MemTableFlusher {
             return Err(Error::invalid_input("Cannot flush empty MemTable"));
         }
 
-        if !memtable.all_flushed_to_wal() {
+        if !memtable.all_flushed_to_wal(durable) {
             return Err(Error::invalid_input(
                 "MemTable has unflushed fragments - WAL flush required first",
             ));
@@ -177,8 +234,7 @@ impl MemTableFlusher {
         let random_hash = generate_random_hash();
         let generation = memtable.generation();
         let gen_folder_name = format!("{}_gen_{}", random_hash, generation);
-        let gen_path =
-            flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation);
+        let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation);
 
         info!(
             "Flushing MemTable generation {} to {} ({} rows, {} batches)",
@@ -190,11 +246,11 @@ impl MemTableFlusher {
 
         let (rows_flushed, deleted) = self.write_data_file(&gen_path, memtable).await?;
 
-        // Persist the within-generation deletion vector so the flushed
-        // generation exposes newest-per-PK on every read path.
+        // Persist the within-generation deletion vector so the
+        // SSTable exposes newest-per-PK on every read path.
         if !deleted.is_empty() {
             let uri = self.path_to_uri(&gen_path);
-            let dataset = Dataset::open(&uri).await?;
+            let dataset = self.open_generation(&uri).await?;
             self.finalize_generation(&dataset, &deleted, None).await?;
         }
 
@@ -222,12 +278,12 @@ impl MemTableFlusher {
             .await?;
 
         info!(
-            "Flushed generation {} for shard {} (manifest version {})",
+            "Flushed SSTable {} for shard {} (manifest version {})",
             generation, self.shard_id, new_manifest.version
         );
 
         Ok(FlushResult {
-            generation: FlushedGeneration {
+            sstable: SsTable {
                 generation,
                 path: gen_folder_name,
             },
@@ -296,13 +352,19 @@ impl MemTableFlusher {
         let reader =
             RecordBatchIterator::new(batches.into_iter().map(Ok), memtable.schema().clone());
 
-        // Use very large max_rows_per_file to ensure 1 fragment per flushed memtable.
-        // Inherit the base dataset's storage version so the flushed generation
+        // Use very large max_rows_per_file to ensure 1 fragment per SSTable.
+        // Inherit the base dataset's storage version so the SSTable
         // matches it (a 2.2 base also fixes the v2.1 miniblock 32 KiB chunk cap
         // that the dense HNSW graph List columns overflow at scale).
         let write_params = WriteParams {
             max_rows_per_file: usize::MAX,
             data_storage_version: Some(self.base_storage_version().await?),
+            // Write the generation through the base's store params + session so it
+            // uses the same store the base was opened with. Adapted for the
+            // generation URI: a path-bound store binding would send this write at
+            // the base table's own path (see [`derived_store_params`]).
+            store_params: self.store_params.as_ref().map(derived_store_params),
+            session: self.session.clone(),
             ..Default::default()
         };
         Dataset::write(reader, &uri, Some(write_params)).await?;
@@ -334,7 +396,7 @@ impl MemTableFlusher {
             let dv = DeletionVector::from(deleted.clone());
             let deletion_file = write_deletion_file(
                 &dataset.base,
-                0, // 1 fragment per flushed generation
+                0, // 1 fragment per SSTable
                 dataset.version().version,
                 &dv,
                 dataset.object_store.as_ref(),
@@ -388,6 +450,7 @@ impl MemTableFlusher {
         epoch: u64,
         index_configs: &[MemIndexConfig],
         covered_wal_entry_position: u64,
+        durable: usize,
     ) -> Result<FlushResult> {
         self.manifest_store.check_fenced(epoch).await?;
 
@@ -395,7 +458,7 @@ impl MemTableFlusher {
             return Err(Error::invalid_input("Cannot flush empty MemTable"));
         }
 
-        if !memtable.all_flushed_to_wal() {
+        if !memtable.all_flushed_to_wal(durable) {
             return Err(Error::invalid_input(
                 "MemTable has unflushed fragments - WAL flush required first",
             ));
@@ -404,8 +467,7 @@ impl MemTableFlusher {
         let random_hash = generate_random_hash();
         let generation = memtable.generation();
         let gen_folder_name = format!("{}_gen_{}", random_hash, generation);
-        let gen_path =
-            flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation);
+        let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation);
 
         info!(
             "Flushing MemTable generation {} with indexes to {} ({} rows, {} batches)",
@@ -420,7 +482,7 @@ impl MemTableFlusher {
         // Open the dataset once for all index building. Dataset::write already
         // created a v1 manifest with the fragment data.
         let uri = self.path_to_uri(&gen_path);
-        let mut dataset = Dataset::open(&uri).await?;
+        let mut dataset = self.open_generation(&uri).await?;
 
         // Collect all index metadata without committing individually.
         // We write a single manifest containing both data and all indexes.
@@ -431,7 +493,7 @@ impl MemTableFlusher {
             .await?;
         if !btree_indexes.is_empty() {
             info!(
-                "Created {} BTree indexes on flushed generation {}",
+                "Created {} BTree indexes on SSTable {}",
                 btree_indexes.len(),
                 generation
             );
@@ -450,7 +512,7 @@ impl MemTableFlusher {
                         .await?
                     else {
                         info!(
-                            "Skipped empty HNSW index '{}' on flushed generation {} (no vectors)",
+                            "Skipped empty HNSW index '{}' on SSTable {} (no vectors)",
                             hnsw_config.name, generation
                         );
                         continue;
@@ -474,7 +536,7 @@ impl MemTableFlusher {
                     all_indexes.push(index_meta);
 
                     info!(
-                        "Created HNSW index '{}' on flushed generation {}",
+                        "Created HNSW index '{}' on SSTable {}",
                         hnsw_config.name, generation
                     );
                 }
@@ -520,12 +582,12 @@ impl MemTableFlusher {
             .await?;
 
         info!(
-            "Flushed generation {} for shard {} (manifest version {})",
+            "Flushed SSTable {} for shard {} (manifest version {})",
             generation, self.shard_id, new_manifest.version
         );
 
         Ok(FlushResult {
-            generation: FlushedGeneration {
+            sstable: SsTable {
                 generation,
                 path: gen_folder_name,
             },
@@ -534,7 +596,7 @@ impl MemTableFlusher {
         })
     }
 
-    /// Create BTree indexes on the flushed dataset (uncommitted).
+    /// Create BTree indexes on the SSTable dataset (uncommitted).
     ///
     /// Returns index metadata without committing to the dataset manifest.
     /// The caller is responsible for writing a single manifest with all indexes.
@@ -602,7 +664,7 @@ impl MemTableFlusher {
     /// keys index the typed value; composite keys index the order-preserving
     /// `Binary` encoded tuple (see [`super::super::index::encode_pk_tuple`]).
     /// Row positions line up 1:1 with the forward-written data file, so they are
-    /// the flushed row ids directly. No-op without a primary-key index.
+    /// the SSTable row ids directly. No-op without a primary-key index.
     async fn create_pk_index(
         &self,
         gen_path: &Path,
@@ -749,8 +811,9 @@ impl MemTableFlusher {
         use std::sync::Arc;
 
         use lance_index::scalar::inverted::{
-            POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1, POSITIONS_LAYOUT_KEY,
-            POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_TAIL_CODEC_KEY, TokenSetFormat,
+            FTS_FORMAT_VERSION_KEY, POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1,
+            POSITIONS_LAYOUT_KEY, POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_BLOCK_SIZE_KEY,
+            POSTING_TAIL_CODEC_KEY, TokenSetFormat,
         };
 
         // Create metadata with params and partitions in schema metadata (this is what InvertedIndex expects)
@@ -766,6 +829,14 @@ impl MemTableFlusher {
                 POSTING_TAIL_CODEC_KEY.to_string(),
                 format_version.posting_tail_codec().as_str().to_string(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_string(),
+                format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_string(),
+                config.params.posting_block_size().to_string(),
+            ),
         ]
         .into_iter()
         .collect::<std::collections::HashMap<_, _>>();
@@ -807,7 +878,7 @@ impl MemTableFlusher {
     /// the existing Lance `IVF_HNSW_SQ` reader path.
     ///
     /// # Arguments
-    /// * `gen_path` - Path to the flushed generation folder
+    /// * `gen_path` - Path to the SSTable folder
     /// * `config` - HNSW index configuration
     /// * `mem_index` - In-memory HNSW index (snapshotted, not consumed)
     ///
@@ -1054,7 +1125,7 @@ impl MemTableFlusher {
         Ok(Some(index_meta))
     }
 
-    /// Update the shard manifest with the new flushed generation.
+    /// Update the shard manifest with the new SSTable.
     async fn update_manifest(
         &self,
         epoch: u64,
@@ -1066,8 +1137,8 @@ impl MemTableFlusher {
 
         self.manifest_store
             .commit_update(epoch, |current| {
-                let mut flushed_generations = current.flushed_generations.clone();
-                flushed_generations.push(FlushedGeneration {
+                let mut sstables = current.sstables.clone();
+                sstables.push(SsTable {
                     generation,
                     path: gen_path.clone(),
                 });
@@ -1079,7 +1150,7 @@ impl MemTableFlusher {
                         .wal_entry_position_last_seen
                         .max(covered_wal_entry_position),
                     current_generation: generation + 1,
-                    flushed_generations,
+                    sstables,
                     ..current.clone()
                 }
             })
@@ -1119,6 +1190,7 @@ mod tests {
     use super::*;
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+    use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V2;
     use std::sync::Arc;
     use tempfile::TempDir;
 
@@ -1185,11 +1257,12 @@ mod tests {
             .await
             .unwrap();
 
-        // Not flushed to WAL yet
-        assert!(!memtable.all_flushed_to_wal());
+        // Nothing is durable yet, so the L0 flush must refuse.
+        let durable = 0;
+        assert!(!memtable.all_flushed_to_wal(durable));
 
         let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
-        let result = flusher.flush(&memtable, epoch, 0).await;
+        let result = flusher.flush(&memtable, epoch, 0, 0).await;
 
         assert!(result.is_err());
         assert!(
@@ -1218,7 +1291,7 @@ mod tests {
         let memtable = MemTable::new(schema, 1, vec![]).unwrap();
 
         let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
-        let result = flusher.flush(&memtable, epoch, 0).await;
+        let result = flusher.flush(&memtable, epoch, 0, 0).await;
 
         assert!(result.is_err());
         assert!(result.unwrap_err().to_string().contains("empty MemTable"));
@@ -1246,8 +1319,8 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
-        assert!(memtable.all_flushed_to_wal());
+        let durable = frag_id + 1;
+        assert!(memtable.all_flushed_to_wal(durable));
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1256,9 +1329,9 @@ mod tests {
             shard_id,
             manifest_store.clone(),
         );
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
-        assert_eq!(result.generation.generation, 1);
+        assert_eq!(result.sstable.generation, 1);
         assert_eq!(result.rows_flushed, 10);
         assert_eq!(result.covered_wal_entry_position, 1);
 
@@ -1267,10 +1340,10 @@ mod tests {
         assert_eq!(updated_manifest.version, 2);
         assert_eq!(updated_manifest.replay_after_wal_entry_position, 1);
         assert_eq!(updated_manifest.current_generation, 2);
-        assert_eq!(updated_manifest.flushed_generations.len(), 1);
+        assert_eq!(updated_manifest.sstables.len(), 1);
     }
 
-    /// A `GenerationWarmer` that counts calls and optionally fails.
+    /// A `SsTableWarmer` that counts calls and optionally fails.
     #[derive(Debug)]
     struct CountingWarmer {
         calls: Arc<std::sync::atomic::AtomicUsize>,
@@ -1278,7 +1351,7 @@ mod tests {
     }
 
     #[async_trait::async_trait]
-    impl GenerationWarmer for CountingWarmer {
+    impl SsTableWarmer for CountingWarmer {
         async fn warm(&self, _path: &str) -> Result<()> {
             self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
             if self.fail {
@@ -1310,10 +1383,10 @@ mod tests {
             .insert(create_test_batch(&schema, 10))
             .await
             .unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let warmer: Arc<dyn GenerationWarmer> = Arc::new(CountingWarmer {
+        let warmer: Arc<dyn SsTableWarmer> = Arc::new(CountingWarmer {
             calls: calls.clone(),
             fail: true,
         });
@@ -1327,9 +1400,9 @@ mod tests {
         )
         .with_warmer(Some(warmer));
         // Flush must succeed despite the warmer erroring.
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
-        assert_eq!(result.generation.generation, 1);
+        assert_eq!(result.sstable.generation, 1);
         assert_eq!(
             calls.load(std::sync::atomic::Ordering::SeqCst),
             1,
@@ -1337,14 +1410,14 @@ mod tests {
         );
         let updated = manifest_store.read_latest().await.unwrap().unwrap();
         assert_eq!(
-            updated.flushed_generations.len(),
+            updated.sstables.len(),
             1,
             "generation still committed after a failed warm"
         );
     }
 
     /// Flushing a generation with within-generation duplicate PKs writes a
-    /// deletion vector so the flushed dataset exposes newest-per-PK on scan.
+    /// deletion vector so the SSTable dataset exposes newest-per-PK on scan.
     #[tokio::test]
     async fn test_flush_writes_dedup_deletion_vector() {
         use futures::TryStreamExt;
@@ -1371,7 +1444,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1380,16 +1453,16 @@ mod tests {
             shard_id,
             manifest_store,
         );
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
         assert_eq!(result.rows_flushed, 5, "all physical rows are written");
 
-        // Scanning the flushed generation must honor the deletion vector and
+        // Scanning the SSTable must honor the deletion vector and
         // return only the newest version of each PK.
         let gen_uri = format!(
             "{}/_mem_wal/{}/{}",
             base_uri.trim_end_matches('/'),
             shard_id,
-            result.generation.path
+            result.sstable.path
         );
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         let batches: Vec<RecordBatch> = dataset
@@ -1436,7 +1509,7 @@ mod tests {
     /// probe by value — including for a within-gen-superseded PK (existence,
     /// not visibility).
     #[tokio::test]
-    async fn flushed_pk_index_sidecar_is_probeable() {
+    async fn sstable_pk_index_sidecar_is_probeable() {
         use lance_core::cache::LanceCache;
         use lance_index::metrics::NoOpMetricsCollector;
         use lance_index::registry::IndexPluginRegistry;
@@ -1474,7 +1547,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1484,7 +1557,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &[], 1)
+            .flush_with_indexes(&memtable, epoch, &[], 1, durable)
             .await
             .unwrap();
 
@@ -1493,7 +1566,7 @@ mod tests {
             .clone()
             .join("_mem_wal")
             .join(shard_id.to_string())
-            .join(result.generation.path.as_str());
+            .join(result.sstable.path.as_str());
         let index_store = Arc::new(LanceIndexStore::new(
             store.clone(),
             pk_index_path(&gen_path),
@@ -1574,7 +1647,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1584,13 +1657,13 @@ mod tests {
             manifest_store.clone(),
         );
         // The plain-flush path — what the writer dispatches to with no indexes.
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
         let gen_path = base_path
             .clone()
             .join("_mem_wal")
             .join(shard_id.to_string())
-            .join(result.generation.path.as_str());
+            .join(result.sstable.path.as_str());
         let index_store = Arc::new(LanceIndexStore::new(
             store.clone(),
             pk_index_path(&gen_path),
@@ -1668,7 +1741,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1678,7 +1751,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
         assert_eq!(result.rows_flushed, 5, "all physical rows are written");
@@ -1687,13 +1760,13 @@ mod tests {
             "{}/_mem_wal/{}/{}",
             base_uri.trim_end_matches('/'),
             shard_id,
-            result.generation.path
+            result.sstable.path
         );
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         assert_eq!(
             dataset.version().version,
             1,
-            "flushed dataset must be a single-version dataset"
+            "SSTable dataset must be a single-version dataset"
         );
 
         // Index half of the combined manifest.
@@ -1800,7 +1873,7 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1810,23 +1883,20 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
-        assert_eq!(result.generation.generation, 1);
+        assert_eq!(result.sstable.generation, 1);
         assert_eq!(result.rows_flushed, 10);
 
-        // Verify the flushed dataset is a single-version dataset with the BTree index
-        let gen_uri = format!(
-            "{}/_mem_wal/{}/{}",
-            base_uri, shard_id, result.generation.path
-        );
+        // Verify the SSTable dataset is a single-version dataset with the BTree index
+        let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path);
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         assert_eq!(
             dataset.version().version,
             1,
-            "flushed dataset must be a single-version dataset"
+            "SSTable dataset must be a single-version dataset"
         );
         let indices = dataset.load_indices().await.unwrap();
 
@@ -1937,7 +2007,7 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1947,30 +2017,27 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
-        assert_eq!(result.generation.generation, 1);
+        assert_eq!(result.sstable.generation, 1);
         assert_eq!(result.rows_flushed, num_vectors);
 
-        // Verify the flushed dataset is a single-version dataset with the HNSW index
-        let gen_uri = format!(
-            "{}/_mem_wal/{}/{}",
-            base_uri, shard_id, result.generation.path
-        );
+        // Verify the SSTable dataset is a single-version dataset with the HNSW index
+        let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path);
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         assert_eq!(
             dataset.version().version,
             1,
-            "flushed dataset must be a single-version dataset"
+            "SSTable dataset must be a single-version dataset"
         );
         let indices = dataset.load_indices().await.unwrap();
 
         assert_eq!(indices.len(), 1);
         assert_eq!(indices[0].name, "vector_hnsw");
 
-        // End-to-end query: pick a row from the flushed dataset, query for
+        // End-to-end query: pick a row from the SSTable dataset, query for
         // it, and verify the index path returns it as the nearest neighbor.
         // This exercises the on-disk HNSW + SQ8 format including the IVF
         // partition routing and the storage_metadata ScalarQuantizationMetadata
@@ -2087,7 +2154,7 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -2097,28 +2164,26 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
-        assert_eq!(result.generation.generation, 1);
+        assert_eq!(result.sstable.generation, 1);
         assert_eq!(result.rows_flushed, 3);
 
-        // Verify the flushed dataset is a single-version dataset with the FTS index
-        let gen_uri = format!(
-            "{}/_mem_wal/{}/{}",
-            base_uri, shard_id, result.generation.path
-        );
+        // Verify the SSTable dataset is a single-version dataset with the FTS index
+        let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path);
         let dataset = Dataset::open(&gen_uri).await.unwrap();
         assert_eq!(
             dataset.version().version,
             1,
-            "flushed dataset must be a single-version dataset"
+            "SSTable dataset must be a single-version dataset"
         );
         let indices = dataset.load_indices().await.unwrap();
 
         assert_eq!(indices.len(), 1);
         assert_eq!(indices[0].name, "text_fts");
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V2 as i32);
 
         // Verify FTS query returns correct results
         // Searching for "hello" should find the first document
@@ -2172,9 +2237,8 @@ mod tests {
         crate::utils::test::assert_plan_node_equals(
             plan,
             "ProjectionExec: expr=[id@2 as id, text@3 as text, _score@1 as _score]
-  Take: ...
-    CoalesceBatchesExec: ...
-      MatchQuery: column=text, query=hello",
+  LanceRead: ..., source=stream(_rowid)
+    MatchQuery: column=text, query=[hello]",
         )
         .await
         .unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
index e1f0d6e689c..e0a06ff85b9 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
@@ -378,7 +378,7 @@ impl ScalarPredicate {
 ///
 /// # Index Visibility Model
 ///
-/// The scanner captures `max_visible_batch_position` from the `IndexStore` at
+/// The scanner captures `visible_count` from the `IndexStore` at
 /// construction time. This frozen visibility ensures queries only see data
 /// that has been indexed, providing consistent results.
 ///
@@ -401,8 +401,8 @@ pub struct MemTableScanner {
     indexes: Arc<IndexStore>,
     schema: SchemaRef,
     /// Frozen visibility captured at scanner construction time.
-    /// This is the `max_visible_batch_position` from the IndexStore.
-    max_visible_batch_position: usize,
+    /// This is the `visible_count` from the IndexStore.
+    visible_count: usize,
     projection: Option<Vec<String>>,
     filter: Option<Expr>,
     limit: Option<usize>,
@@ -427,7 +427,7 @@ pub struct MemTableScanner {
 impl MemTableScanner {
     /// Create a new scanner.
     ///
-    /// Captures `max_visible_batch_position` from the `IndexStore` at construction
+    /// Captures `visible_count` from the `IndexStore` at construction
     /// time to ensure consistent query visibility.
     ///
     /// # Arguments
@@ -439,13 +439,13 @@ impl MemTableScanner {
         // Snapshot the visibility cursor at construction time. The cursor is
         // advanced by `flush_from_batch_store` after the WAL append succeeds,
         // so this snapshot reflects WAL-durable data.
-        let max_visible_batch_position = indexes.max_visible_batch_position();
+        let visible_count = indexes.visible_count();
 
         Self {
             batch_store,
             indexes,
             schema,
-            max_visible_batch_position,
+            visible_count,
             projection: None,
             filter: None,
             limit: None,
@@ -504,12 +504,12 @@ impl MemTableScanner {
         self
     }
 
-    /// The `max_visible_batch_position` snapshot this scanner latched at
+    /// The `visible_count` snapshot this scanner latched at
     /// construction. A downstream recency filter must key on this same snapshot
     /// (not a fresh read of the IndexStore watermark, which a concurrent append
     /// could have advanced) so it stays consistent with the rows the search saw.
-    pub fn max_visible_batch_position(&self) -> usize {
-        self.max_visible_batch_position
+    pub fn visible_count(&self) -> usize {
+        self.visible_count
     }
 
     /// Include the _rowaddr column in output.
@@ -971,7 +971,7 @@ impl MemTableScanner {
 
         let scan = MemTableScanExec::with_filter(
             self.batch_store.clone(),
-            self.max_visible_batch_position,
+            self.visible_count,
             projection_indices,
             self.output_schema(),
             self.schema.clone(),
@@ -1033,7 +1033,7 @@ impl MemTableScanner {
 
         Ok(Arc::new(MemTableDedupScanExec::new(
             self.batch_store.clone(),
-            self.max_visible_batch_position,
+            self.visible_count,
             projection_indices,
             self.output_schema(),
             pk_indices,
@@ -1056,7 +1056,7 @@ impl MemTableScanner {
             return self.plan_full_scan().await;
         }
 
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
 
         let index_exec = BTreeIndexExec::new(
@@ -1095,7 +1095,7 @@ impl MemTableScanner {
     }
 
     async fn plan_vector_search(&self, query: &VectorQuery) -> Result<Arc<dyn ExecutionPlan>> {
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
         let base_schema = self.base_output_schema();
         let filter_predicate = self.filter_predicate()?;
@@ -1154,7 +1154,7 @@ impl MemTableScanner {
             return self.empty_fts_plan();
         }
 
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
         let filter_predicate = self.filter_predicate()?;
         if let Some(pk_columns) = &self.pk_columns {
@@ -1426,7 +1426,7 @@ mod tests {
         let indexes = Arc::new(index_store);
         let scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
         let result = scanner.try_into_batch().await.unwrap();
-        // max_visible_batch_position is 1, so we see batches 0 and 1 (20 rows)
+        // visible_count is 1, so we see batches 0 and 1 (20 rows)
         assert_eq!(result.num_rows(), 20);
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
index 2a593a0f215..9a476edb81b 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
@@ -36,7 +36,7 @@ pub use vector::VectorIndexExec;
 pub(super) fn newest_pk_positions(
     batch_store: &BatchStore,
     pk_columns: &[String],
-    max_visible_batch_position: usize,
+    visible_count: usize,
     max_visible_row: u64,
 ) -> DataFusionResult<HashSet<u64>> {
     let mut newest: HashMap<Vec<ScalarValue>, u64> = HashMap::new();
@@ -46,7 +46,7 @@ pub(super) fn newest_pk_positions(
         if n == 0 {
             continue;
         }
-        if batch_position > max_visible_batch_position {
+        if batch_position >= visible_count {
             current_row += n as u64;
             continue;
         }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
index 8a239605b50..e852d2ad917 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
@@ -11,7 +11,6 @@
 //! or new rows in the window between commit and next memtable rotation), this
 //! exec keeps KNN correct by computing exact distances row-by-row.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -40,7 +39,7 @@ use crate::dataset::mem_wal::write::BatchStore;
 /// Distance metric used when [`VectorQuery::distance_type`] is `None`. The
 /// indexed path defers to the index's own metric, but with no index there is
 /// no inherent default — L2 matches what most callers configure and what the
-/// flushed/base arms use when re-ranking unindexed candidates.
+/// SSTable/base arms use when re-ranking unindexed candidates.
 const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2;
 
 /// Brute-force KNN over an active memtable without an HNSW. Produces the same
@@ -48,7 +47,7 @@ const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2;
 pub struct MemTableBruteForceVectorExec {
     batch_store: Arc<BatchStore>,
     query: VectorQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -69,10 +68,7 @@ impl Debug for MemTableBruteForceVectorExec {
         f.debug_struct("MemTableBruteForceVectorExec")
             .field("column", &self.query.column)
             .field("k", &self.query.k)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .finish()
     }
@@ -85,7 +81,7 @@ impl MemTableBruteForceVectorExec {
     pub fn new(
         batch_store: Arc<BatchStore>,
         query: VectorQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -111,7 +107,7 @@ impl MemTableBruteForceVectorExec {
         Ok(Self {
             batch_store,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -160,7 +156,7 @@ impl MemTableBruteForceVectorExec {
         Ok(Some(mask))
     }
 
-    /// Last row position visible under `max_visible_batch_position`, or `None`
+    /// Last row position visible under `visible_count`, or `None`
     /// if no batches are visible. Identical to `VectorIndexExec`'s helper so
     /// both arms cut at the same MVCC boundary.
     fn compute_max_visible_row(&self) -> Option<u64> {
@@ -169,7 +165,7 @@ impl MemTableBruteForceVectorExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
@@ -225,7 +221,7 @@ impl MemTableBruteForceVectorExec {
                 newest_pk_positions(
                     &self.batch_store,
                     pk_columns,
-                    self.max_visible_batch_position,
+                    self.visible_count,
                     max_visible_row,
                 )
                 .map_err(|e| Error::invalid_input(e.to_string()))?,
@@ -245,7 +241,7 @@ impl MemTableBruteForceVectorExec {
             if n == 0 {
                 continue;
             }
-            if batch_position > self.max_visible_batch_position {
+            if batch_position >= self.visible_count {
                 current_row += n as u64;
                 continue;
             }
@@ -426,10 +422,6 @@ impl ExecutionPlan for MemTableBruteForceVectorExec {
         "MemTableBruteForceVectorExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -466,12 +458,12 @@ impl ExecutionPlan for MemTableBruteForceVectorExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.k),
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -615,7 +607,7 @@ mod tests {
             MemTableBruteForceVectorExec::new(
                 store,
                 query,
-                /* max_visible_batch_position = */ usize::MAX,
+                /* visible_count = */ usize::MAX,
                 None,
                 schema,
                 false,
@@ -668,7 +660,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn respects_max_visible_batch_position() {
+    async fn respects_indexed_count() {
         // Two batches of two rows. Freeze at batch 0 — only ids 0,1 are
         // visible candidates; the (closer) ids 2,3 in batch 1 are excluded.
         let schema = make_schema();
@@ -678,7 +670,7 @@ mod tests {
         let query = query_for([0.0, 0.0], 4);
         let exec = Arc::new(
             MemTableBruteForceVectorExec::new(
-                store, query, /* max_visible_batch_position = */ 0, None, schema, false,
+                store, query, /* visible_count = */ 1, None, schema, false,
             )
             .expect("ctor"),
         );
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
index fed61698fab..01781d07ad8 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
@@ -3,7 +3,6 @@
 
 //! BTreeIndexExec - BTree index queries with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -31,7 +30,7 @@ pub struct BTreeIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     predicate: ScalarPredicate,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -48,10 +47,7 @@ impl Debug for BTreeIndexExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("BTreeIndexExec")
             .field("predicate", &self.predicate)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .field("with_row_address", &self.with_row_address)
             .field("column", &self.column)
@@ -67,7 +63,7 @@ impl BTreeIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with BTree indexes
     /// * `predicate` - Scalar predicate to apply
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -77,7 +73,7 @@ impl BTreeIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         predicate: ScalarPredicate,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         with_row_id: bool,
@@ -103,7 +99,7 @@ impl BTreeIndexExec {
             batch_store,
             indexes,
             predicate,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -114,7 +110,7 @@ impl BTreeIndexExec {
         })
     }
 
-    /// Compute the maximum visible row position based on max_visible_batch_position.
+    /// Compute the maximum visible row position based on visible_count.
     /// Returns None if no batches are visible.
     fn compute_max_visible_row(&self) -> Option<u64> {
         let mut max_visible_row_exclusive: u64 = 0;
@@ -122,7 +118,7 @@ impl BTreeIndexExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
@@ -312,10 +308,6 @@ impl ExecutionPlan for BTreeIndexExec {
         "BTreeIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -358,13 +350,13 @@ impl ExecutionPlan for BTreeIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         // We can't know the exact count without querying the index
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -434,7 +426,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0, // max_visible_batch_position (batch at position 0)
+            1, // visible_count (batch at position 0)
             None,
             schema,
             false,
@@ -478,7 +470,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0,
+            1,
             None,
             schema,
             false,
@@ -523,7 +515,7 @@ mod tests {
             batch_store.clone(),
             indexes.clone(),
             predicate.clone(),
-            0,
+            1,
             None,
             schema.clone(),
             false,
@@ -543,7 +535,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            1,
+            2,
             None,
             schema,
             false,
@@ -592,7 +584,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0,
+            1,
             None,
             schema_with_rowid.clone(),
             true,
@@ -656,7 +648,7 @@ mod tests {
                 batch_store.clone(),
                 indexes.clone(),
                 predicate.clone(),
-                0,
+                1,
                 None,
                 schema.clone(),
                 false,
@@ -684,7 +676,7 @@ mod tests {
                 batch_store,
                 indexes,
                 predicate,
-                0,
+                1,
                 None,
                 schema_with_rowid,
                 true,
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
index ba5947e4b12..66d73becb69 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
@@ -16,7 +16,6 @@
 //! forward-aligned mask. A single `filter_record_batch` over the original
 //! batch then emits the survivors with no per-column reverse copy.
 
-use std::any::Any;
 use std::collections::HashSet;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
@@ -44,7 +43,7 @@ use crate::dataset::mem_wal::write::BatchStore;
 /// that satisfy the (optional) predicate. See the module doc.
 pub struct MemTableDedupScanExec {
     batch_store: Arc<BatchStore>,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     /// Column indices to project (into the source schema).
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
@@ -62,10 +61,7 @@ pub struct MemTableDedupScanExec {
 impl Debug for MemTableDedupScanExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MemTableDedupScanExec")
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("projection", &self.projection)
             .field("pk_indices", &self.pk_indices)
             .field("with_row_address", &self.with_row_address)
@@ -79,7 +75,7 @@ impl MemTableDedupScanExec {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         pk_indices: Vec<usize>,
@@ -97,7 +93,7 @@ impl MemTableDedupScanExec {
 
         Self {
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             pk_indices,
@@ -155,10 +151,6 @@ impl ExecutionPlan for MemTableDedupScanExec {
         "MemTableDedupScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -188,7 +180,7 @@ impl ExecutionPlan for MemTableDedupScanExec {
         // back-to-front below.
         let mut batches = self
             .batch_store
-            .visible_batches_with_offsets(self.max_visible_batch_position);
+            .visible_batches_with_offsets(self.visible_count);
         batches.reverse();
 
         let projection = self.projection.clone();
@@ -280,12 +272,12 @@ impl ExecutionPlan for MemTableDedupScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -347,7 +339,7 @@ mod tests {
     /// Run the exec and collect (id -> (value, rowaddr)).
     async fn run(
         store: Arc<BatchStore>,
-        max_visible: usize,
+        visible_count: usize,
         filter: Option<Expr>,
     ) -> HashMap<i32, (Option<i32>, u64)> {
         let filter_predicate = filter.map(|expr| {
@@ -358,7 +350,7 @@ mod tests {
         let filter_expr = None;
         let exec = MemTableDedupScanExec::new(
             store,
-            max_visible,
+            visible_count,
             None,
             output_schema(),
             vec![0],
@@ -392,11 +384,11 @@ mod tests {
         // id=10 inserted (100) then updated to NULL, all in one batch.
         store.append(batch(&[(10, Some(100)), (10, None)])).unwrap();
 
-        let no_filter = run(store.clone(), 0, None).await;
+        let no_filter = run(store.clone(), 1, None).await;
         assert_eq!(no_filter.len(), 1);
         assert_eq!(no_filter[&10].0, None, "newest version of id=10 is NULL");
 
-        let not_null = run(store, 0, Some(col("value").is_not_null())).await;
+        let not_null = run(store, 1, Some(col("value").is_not_null())).await;
         assert!(
             !not_null.contains_key(&10),
             "id=10 newest is NULL; the stale value=100 must not leak under value IS NOT NULL"
@@ -413,7 +405,7 @@ mod tests {
         store.append(batch(&[(20, Some(999)), (30, None)])).unwrap();
 
         // No filter: newest per PK = {10:NULL@2, 20:999@3, 30:NULL@4}.
-        let all = run(store.clone(), 1, None).await;
+        let all = run(store.clone(), 2, None).await;
         assert_eq!(all.len(), 3);
         assert_eq!(all[&10], (None, 2));
         assert_eq!(all[&20], (Some(999), 3));
@@ -421,7 +413,7 @@ mod tests {
 
         // value IS NOT NULL: only id=20 (newest 999) survives; 10 and 30 are
         // newest-NULL so they must be absent (no stale leak).
-        let not_null = run(store, 1, Some(col("value").is_not_null())).await;
+        let not_null = run(store, 2, Some(col("value").is_not_null())).await;
         assert_eq!(not_null.len(), 1);
         assert_eq!(not_null[&20], (Some(999), 3));
     }
@@ -434,7 +426,7 @@ mod tests {
         // id=40 inserted NULL then updated to 400 (newest non-NULL).
         store.append(batch(&[(40, None), (40, Some(400))])).unwrap();
 
-        let is_null = run(store, 0, Some(col("value").is_null())).await;
+        let is_null = run(store, 1, Some(col("value").is_null())).await;
         assert!(
             !is_null.contains_key(&40),
             "id=40 newest is 400; the stale NULL must not leak under value IS NULL"
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
index 364261c3276..61e8ac85efc 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
@@ -3,7 +3,6 @@
 
 //! FtsIndexExec - Full-text search with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -48,14 +47,14 @@ pub struct FtsIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     query: FtsQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
     /// Pre-computed batch ranges for O(log n) lookup.
     batch_ranges: Vec<BatchRange>,
-    /// Maximum visible row position based on max_visible_batch_position (None if nothing visible).
+    /// Maximum visible row position based on visible_count (None if nothing visible).
     max_visible_row: Option<u64>,
     /// Whether to include _rowid column (row position) in output.
     with_row_id: bool,
@@ -73,10 +72,7 @@ impl Debug for FtsIndexExec {
         f.debug_struct("FtsIndexExec")
             .field("column", &self.query.column)
             .field("query_type", &self.query.query_type)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .finish()
     }
@@ -90,7 +86,7 @@ impl FtsIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with FTS indexes
     /// * `query` - FTS query parameters
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `base_schema` - Schema before adding score column (and _rowid if with_row_id)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -98,7 +94,7 @@ impl FtsIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         query: FtsQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -119,9 +115,9 @@ impl FtsIndexExec {
             .map(|f| f.as_ref().clone())
             .collect();
         // `_score` is nullable here to stay schema-compatible with
-        // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/flushed
+        // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/SSTable
         // FTS exec nodes emit). The LSM `full_text_search` planner unions the
-        // active arm with base/flushed arms; UnionExec requires schema equality
+        // active arm with base/SSTable arms; UnionExec requires schema equality
         // including nullability. The actual emitted column is always populated.
         fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true));
         if with_row_id {
@@ -149,7 +145,7 @@ impl FtsIndexExec {
                 end: batch_end,
                 batch_id,
             });
-            if batch_id <= max_visible_batch_position {
+            if batch_id < visible_count {
                 max_visible_row_exclusive = batch_end as u64;
             }
             current_row = batch_end;
@@ -166,7 +162,7 @@ impl FtsIndexExec {
             batch_store,
             indexes,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -459,7 +455,7 @@ impl FtsIndexExec {
             Some(newest_pk_positions(
                 &self.batch_store,
                 pk_columns,
-                self.max_visible_batch_position,
+                self.visible_count,
                 max_visible_row,
             )?)
         };
@@ -529,10 +525,6 @@ impl ExecutionPlan for FtsIndexExec {
         "FtsIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -578,12 +570,12 @@ impl ExecutionPlan for FtsIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -646,7 +638,7 @@ mod tests {
 
         let query = FtsQuery::match_query("text", "hello");
 
-        let exec = FtsIndexExec::new(batch_store, indexes, query, 0, None, schema, false).unwrap();
+        let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap();
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -687,7 +679,7 @@ mod tests {
             batch_store.clone(),
             indexes.clone(),
             query.clone(),
-            0,
+            1,
             None,
             schema.clone(),
             false,
@@ -702,7 +694,7 @@ mod tests {
         assert_eq!(total_rows, 2); // "hello" in batch1 docs 0 and 2
 
         // Query with max_visible=1 should see both batches
-        let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap();
+        let exec = FtsIndexExec::new(batch_store, indexes, query, 2, None, schema, false).unwrap();
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
index c56e960048d..6126f392f1f 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
@@ -3,7 +3,6 @@
 
 //! MemTableScanExec - Full table scan with MVCC visibility filtering.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -31,12 +30,12 @@ pub const ROW_ADDRESS_COLUMN: &str = "_rowaddr";
 /// ExecutionPlan node that scans all visible batches from a MemTable.
 ///
 /// This node implements visibility filtering, returning only batches
-/// where `batch_position <= max_visible_batch_position`.
+/// where `batch_position <= visible_count`.
 ///
 /// Supports filter pushdown for efficient predicate evaluation during scan.
 pub struct MemTableScanExec {
     batch_store: Arc<BatchStore>,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     /// Schema of the source data (before projection), used for filter evaluation.
@@ -56,10 +55,7 @@ pub struct MemTableScanExec {
 impl Debug for MemTableScanExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MemTableScanExec")
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("projection", &self.projection)
             .field("with_row_id", &self.with_row_id)
             .field("with_row_address", &self.with_row_address)
@@ -74,20 +70,20 @@ impl MemTableScanExec {
     /// # Arguments
     ///
     /// * `batch_store` - Lock-free batch store containing data
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `with_row_id` - Whether to include _rowid column (row position)
     pub fn new(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         with_row_id: bool,
     ) -> Self {
         Self::with_filter(
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema.clone(),
             output_schema,
@@ -103,7 +99,7 @@ impl MemTableScanExec {
     /// # Arguments
     ///
     /// * `batch_store` - Lock-free batch store containing data
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `source_schema` - Schema of source data (before projection), used for filter evaluation
@@ -114,7 +110,7 @@ impl MemTableScanExec {
     #[allow(clippy::too_many_arguments)]
     pub fn with_filter(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         source_schema: SchemaRef,
@@ -132,7 +128,7 @@ impl MemTableScanExec {
 
         Self {
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             source_schema,
@@ -194,10 +190,6 @@ impl ExecutionPlan for MemTableScanExec {
         "MemTableScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -226,7 +218,7 @@ impl ExecutionPlan for MemTableScanExec {
         // Get visible batches with their row offsets
         let batches_with_offsets = self
             .batch_store
-            .visible_batches_with_offsets(self.max_visible_batch_position);
+            .visible_batches_with_offsets(self.visible_count);
 
         let projection = self.projection.clone();
         let schema = self.output_schema.clone();
@@ -339,14 +331,14 @@ impl ExecutionPlan for MemTableScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         // Report statistics as Absent to avoid DataFusion analysis bugs
         // with selectivity calculation on in-memory tables.
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -399,7 +391,7 @@ mod tests {
         batch_store.append(batch).unwrap();
 
         // Batch is at position 0, max_visible=0 means position 0 is visible
-        let exec = MemTableScanExec::new(batch_store, 0, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -425,8 +417,8 @@ mod tests {
             .append(create_test_batch(&schema, 20, 10))
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 are visible (2 batches)
-        let exec = MemTableScanExec::new(batch_store.clone(), 1, None, schema.clone(), false);
+        // visible_count=1 means positions 0 and 1 are visible (2 batches)
+        let exec = MemTableScanExec::new(batch_store.clone(), 2, None, schema.clone(), false);
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
         let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
@@ -447,7 +439,7 @@ mod tests {
         // Project only "id" column (index 0)
         let projected_schema =
             Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        let exec = MemTableScanExec::new(batch_store, 0, Some(vec![0]), projected_schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, Some(vec![0]), projected_schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -464,7 +456,7 @@ mod tests {
         let batch_store = Arc::new(BatchStore::with_capacity(100));
 
         // Empty store with max_visible=0 should return no batches
-        let exec = MemTableScanExec::new(batch_store, 0, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -486,7 +478,7 @@ mod tests {
             .unwrap();
 
         // max_visible=1 means positions 0 and 1 are visible
-        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 2, None, schema, false);
 
         let stats = exec.partition_statistics(None).unwrap();
         // Statistics are Absent to avoid DataFusion analysis bugs
@@ -513,7 +505,7 @@ mod tests {
             Field::new("_rowid", DataType::UInt64, true),
         ]));
 
-        let exec = MemTableScanExec::new(batch_store, 1, None, schema_with_rowid, true);
+        let exec = MemTableScanExec::new(batch_store, 2, None, schema_with_rowid, true);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
index c3453db68a2..0c8f4c27ff9 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
@@ -3,7 +3,6 @@
 
 //! VectorIndexExec - HNSW vector search with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -35,7 +34,7 @@ pub struct VectorIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     query: VectorQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -59,10 +58,7 @@ impl Debug for VectorIndexExec {
         if let Some(metric) = &self.query.distance_type {
             debug.field("distance_type", metric);
         }
-        debug.field(
-            "max_visible_batch_position",
-            &self.max_visible_batch_position,
-        );
+        debug.field("visible_count", &self.visible_count);
         debug.field("with_row_id", &self.with_row_id);
         debug.finish()
     }
@@ -76,7 +72,7 @@ impl VectorIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with HNSW vector indexes
     /// * `query` - Vector query parameters
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `base_schema` - Schema after projection (will add _distance column, and _rowid if with_row_id)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -84,7 +80,7 @@ impl VectorIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         query: VectorQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -120,7 +116,7 @@ impl VectorIndexExec {
             batch_store,
             indexes,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -129,9 +125,9 @@ impl VectorIndexExec {
         })
     }
 
-    /// Compute the maximum visible row position based on max_visible_batch_position.
+    /// Compute the maximum visible row position based on visible_count.
     ///
-    /// Returns the last row position that is visible at the given max_visible_batch_position,
+    /// Returns the last row position that is visible at the given visible_count,
     /// or None if no batches are visible.
     fn compute_max_visible_row(&self) -> Option<u64> {
         let mut max_visible_row_exclusive: u64 = 0;
@@ -139,7 +135,7 @@ impl VectorIndexExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
@@ -310,10 +306,6 @@ impl ExecutionPlan for VectorIndexExec {
         "VectorIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -355,12 +347,12 @@ impl ExecutionPlan for VectorIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.k),
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/scanner.rs b/rust/lance/src/dataset/mem_wal/scanner.rs
index f1d84611e04..907250a1cb1 100644
--- a/rust/lance/src/dataset/mem_wal/scanner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner.rs
@@ -5,8 +5,8 @@
 //!
 //! This module provides scanners that read from multiple data sources
 //! in an LSM tree architecture:
-//! - Base table (merged data)
-//! - Flushed MemTables (persisted but not yet merged)
+//! - Base table (compacted data)
+//! - SSTables (persisted but not yet compacted)
 //! - Active MemTable (in-memory buffer)
 //!
 //! The scanner handles deduplication by primary key, keeping the newest
@@ -41,11 +41,11 @@ mod builder;
 mod collector;
 mod data_source;
 pub mod exec;
-pub(crate) mod flushed_cache;
 mod fts_search;
 mod planner;
 mod point_lookup;
 mod projection;
+pub(crate) mod sstable_cache;
 mod vector_search;
 
 pub use block_list::write_pk_sidecar;
@@ -53,13 +53,11 @@ pub use builder::LsmScanner;
 pub use collector::{
     ActiveMemTableRef, InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector,
 };
-pub use data_source::{
-    FlushedGeneration, FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot,
-};
-pub use flushed_cache::{DatasetCache, FlushedMemTableCache, GenerationWarmer};
+pub use data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot, SsTable};
 pub use fts_search::{LsmFtsSearchPlanner, SCORE_COLUMN};
 pub use point_lookup::LsmPointLookupPlanner;
 pub use projection::DISTANCE_COLUMN;
+pub use sstable_cache::{DatasetCache, SsTableCache, SsTableWarmer};
 pub use vector_search::LsmVectorSearchPlanner;
 
 /// Parse a SQL filter expression against a MemWAL source schema.
diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
index 69d16930888..91fdda8c65f 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
@@ -5,7 +5,7 @@
 //!
 //! A generation's membership is a [`GenMembership`]: in-memory generations
 //! (active / frozen) are probed by value against their maintained primary-key
-//! index (no per-query set), while flushed generations are probed against their
+//! index (no per-query set), while SSTables are probed against their
 //! standalone on-disk PK BTree (the sidecar written at flush, opened by path).
 //! Probing is batched — [`GenMembership::contains_keys`] tests a whole batch of
 //! keys per generation in one pass. Each source gets a `Vec<GenMembership>` of
@@ -32,11 +32,12 @@ use lance_index::scalar::{
 use uuid::Uuid;
 
 use super::data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration};
-use super::flushed_cache::{DatasetCache, open_flushed_dataset};
+use super::sstable_cache::{DatasetCache, open_sstable};
 use crate::dataset::mem_wal::index::encode_pk_tuple;
 use crate::dataset::mem_wal::util::PK_INDEX_DIR;
 use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Default-plugin registry, used only to load the standalone PK BTree by its
 /// `BTreeIndexDetails` type. Built once.
@@ -54,7 +55,7 @@ pub enum GenMembership {
         /// Inclusive visible row watermark; `None` when no rows are visible.
         max_visible_row: Option<u64>,
     },
-    /// Probe the flushed generation's standalone on-disk PK BTree.
+    /// Probe the SSTable's standalone on-disk PK BTree.
     OnDisk(Arc<dyn ScalarIndex>),
 }
 
@@ -110,7 +111,7 @@ impl GenMembership {
     }
 
     /// Whether this generation has no (visible) membership — used to skip adding
-    /// an empty blocked set. A flushed generation always has rows (flush rejects
+    /// an empty blocked set. An SSTable always has rows (flush rejects
     /// an empty memtable), so it is never empty.
     fn is_empty(&self) -> bool {
         match self {
@@ -157,7 +158,8 @@ type ShardGenSets = HashMap<Uuid, Vec<(LsmGeneration, GenMembership)>>;
 pub async fn compute_source_block_lists(
     sources: &[LsmDataSource],
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<dyn DatasetCache>>,
+    store_params: Option<&ObjectStoreParams>,
+    sstable_cache: Option<&Arc<dyn DatasetCache>>,
 ) -> Result<SourceBlockLists> {
     // Membership per non-base source, grouped by shard (generations are
     // per-shard, so supersession is within-shard only).
@@ -165,7 +167,7 @@ pub async fn compute_source_block_lists(
     let mut has_base = false;
     // Flushed PK-BTree opens are cold S3 reads; overlap them with
     // `try_join_all`. Order is irrelevant — gens are sorted per-shard below.
-    let mut flushed_loads = Vec::new();
+    let mut sstable_loads = Vec::new();
     for source in sources {
         match source {
             LsmDataSource::BaseTable { .. } => has_base = true,
@@ -182,18 +184,18 @@ pub async fn compute_source_block_lists(
                     .or_default()
                     .push((*generation, membership));
             }
-            LsmDataSource::FlushedMemTable {
+            LsmDataSource::SsTable {
                 path,
                 shard_id,
                 generation,
                 ..
-            } => flushed_loads.push(async move {
-                let index = open_pk_index(path, session, flushed_cache).await?;
+            } => sstable_loads.push(async move {
+                let index = open_pk_index(path, session, store_params, sstable_cache).await?;
                 Ok::<_, Error>((*shard_id, *generation, GenMembership::OnDisk(index)))
             }),
         }
     }
-    for (shard_id, generation, membership) in futures::future::try_join_all(flushed_loads).await? {
+    for (shard_id, generation, membership) in futures::future::try_join_all(sstable_loads).await? {
         by_shard
             .entry(shard_id)
             .or_default()
@@ -225,7 +227,7 @@ pub async fn compute_source_block_lists(
 
 /// The fresh-tier block-list: one [`GenMembership`] per generation that shadows
 /// the base table — active + frozen memtables (probed against their index) and
-/// flushed generations (probed against their on-disk PK BTree). A base/external
+/// SSTables (probed against their on-disk PK BTree). A base/external
 /// reader can test any PK against these (via [`GenMembership::contains`]) to
 /// decide whether the fresh tier shadows it. The base source, if present, is
 /// skipped (it is what gets shadowed).
@@ -238,14 +240,15 @@ pub async fn compute_source_block_lists(
 pub async fn fresh_tier_block_list(
     sources: &[LsmDataSource],
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<dyn DatasetCache>>,
+    store_params: Option<&ObjectStoreParams>,
+    sstable_cache: Option<&Arc<dyn DatasetCache>>,
     watermarks: Option<&HashMap<Uuid, FreshTierWatermark>>,
 ) -> Result<Vec<GenMembership>> {
     // Membership per source, in source order (`None` = skipped). Flushed
     // PK-BTree opens are cold S3 reads, so collect them tagged with their slot
     // and overlap with `try_join_all` rather than opening one at a time.
     let mut slots: Vec<Option<GenMembership>> = Vec::with_capacity(sources.len());
-    let mut flushed_loads = Vec::new();
+    let mut sstable_loads = Vec::new();
     for source in sources {
         match source {
             LsmDataSource::BaseTable { .. } => slots.push(None),
@@ -278,7 +281,7 @@ pub async fn fresh_tier_block_list(
                 };
                 slots.push(membership);
             }
-            LsmDataSource::FlushedMemTable {
+            LsmDataSource::SsTable {
                 path,
                 shard_id,
                 generation,
@@ -298,15 +301,16 @@ pub async fn fresh_tier_block_list(
                 } else {
                     let slot = slots.len();
                     slots.push(None);
-                    flushed_loads.push(async move {
-                        let index = open_pk_index(path, session, flushed_cache).await?;
+                    sstable_loads.push(async move {
+                        let index =
+                            open_pk_index(path, session, store_params, sstable_cache).await?;
                         Ok::<_, Error>((slot, GenMembership::OnDisk(index)))
                     });
                 }
             }
         }
     }
-    for (slot, membership) in futures::future::try_join_all(flushed_loads).await? {
+    for (slot, membership) in futures::future::try_join_all(sstable_loads).await? {
         slots[slot] = Some(membership);
     }
     Ok(slots
@@ -324,7 +328,7 @@ fn in_memory_membership(
     batch_store: &Arc<BatchStore>,
     index_store: &Arc<IndexStore>,
 ) -> GenMembership {
-    let max_visible_row = batch_store.max_visible_row(index_store.max_visible_batch_position());
+    let max_visible_row = batch_store.max_visible_row(index_store.visible_count());
     GenMembership::InMemory {
         index_store: index_store.clone(),
         max_visible_row,
@@ -342,20 +346,21 @@ fn bounded_in_memory_membership(
     index_store: &Arc<IndexStore>,
     batch_count: u64,
 ) -> GenMembership {
-    let max_visible_row = batch_count
-        .checked_sub(1)
-        .and_then(|last_batch| batch_store.max_visible_row(last_batch as usize));
+    // `batch_count` is already an exclusive count, and so is what
+    // `max_visible_row` takes, so it passes straight through: no
+    // count-to-inclusive-position conversion, and `0` yields `None` on its own.
+    let max_visible_row = batch_store.max_visible_row(batch_count as usize);
     GenMembership::InMemory {
         index_store: index_store.clone(),
         max_visible_row,
     }
 }
 
-/// Open the standalone PK BTree at `{flushed gen}/_pk_index` for one flushed
-/// generation. Reuses the flushed dataset's (session-configured) object store
+/// Open the standalone PK BTree at `{SSTable gen}/_pk_index` for one
+/// SSTable. Reuses the SSTable dataset's (session-configured) object store
 /// and **its index cache**, then loads the sidecar directly by path through the
 /// BTree plugin — it is not a manifest index. The opened index and its pages
-/// are cached in the session's index cache (keyed by the immutable flushed
+/// are cached in the session's index cache (keyed by the immutable SSTable
 /// path), so repeated probes reuse them with no separate cache path and no
 /// upfront scan; concurrent first-opens may each load before the cache fills.
 /// A stable cache UUID for a non-manifest index identified only by its path.
@@ -379,12 +384,13 @@ fn path_cache_uuid(path: &str) -> Uuid {
 async fn open_pk_index(
     path: &str,
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<dyn DatasetCache>>,
+    store_params: Option<&ObjectStoreParams>,
+    sstable_cache: Option<&Arc<dyn DatasetCache>>,
 ) -> Result<Arc<dyn ScalarIndex>> {
-    let dataset = open_flushed_dataset(path, session, flushed_cache, None).await?;
-    // Namespace the session index cache by the (immutable) flushed path so this
+    let dataset = open_sstable(path, session, store_params, sstable_cache, None).await?;
+    // Namespace the session index cache by the (immutable) SSTable path so this
     // sidecar's pages live alongside every other index instead of a bespoke
-    // cache. `fri_uuid` is None — flushed generations carry no fragment-reuse.
+    // cache. `fri_uuid` is None — SSTables carry no fragment-reuse.
     let index_cache = dataset.index_cache.for_index(&path_cache_uuid(path), None);
     let index_dir = dataset.base.clone().join(PK_INDEX_DIR);
     let store: Arc<dyn ScalarIndexStore> = Arc::new(LanceIndexStore::new(
@@ -410,13 +416,13 @@ async fn open_pk_index(
     Ok(index)
 }
 
-/// Write a flushed generation's standalone PK sidecar at `{uri}/_pk_index` from
+/// Write an SSTable's standalone PK sidecar at `{uri}/_pk_index` from
 /// `batches`, mirroring what flush does in production. `pk_columns` are the
 /// primary-key column names (field ids are synthesized by position — `insert`
 /// resolves columns by name). A no-op when no batch carries the PK columns.
 ///
 /// Used by Rust scanner tests and by the Python test-support binding to stage
-/// faithful flushed generations (a flushed dataset alone, with no sidecar, is
+/// faithful SSTables (an SSTable dataset alone, with no sidecar, is
 /// not a state production ever produces).
 pub async fn write_pk_sidecar(
     uri: &str,
@@ -534,7 +540,7 @@ mod tests {
             active_source(shard, 1, &[3]),
         ];
 
-        let memberships = fresh_tier_block_list(&sources, None, None, None)
+        let memberships = fresh_tier_block_list(&sources, None, None, None, None)
             .await
             .unwrap();
 
@@ -555,7 +561,7 @@ mod tests {
             active_source(shard, 2, &[1, 2]),
         ];
 
-        let blocked = Box::pin(compute_source_block_lists(&sources, None, None))
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
             .await
             .unwrap();
 
@@ -591,7 +597,7 @@ mod tests {
             active_source(Uuid::new_v4(), 1, &[1, 2]),
         ];
 
-        let blocked = Box::pin(compute_source_block_lists(&sources, None, None))
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
             .await
             .unwrap();
 
@@ -619,7 +625,7 @@ mod tests {
             active_source(b, 2, &[2]),
         ];
 
-        let blocked = Box::pin(compute_source_block_lists(&sources, None, None))
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
             .await
             .unwrap();
 
@@ -670,7 +676,7 @@ mod tests {
             generation: LsmGeneration::memtable(2),
         };
 
-        let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None))
+        let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None, None))
             .await
             .unwrap();
 
@@ -708,7 +714,7 @@ mod tests {
         )]
         .into_iter()
         .collect();
-        let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks))
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks))
             .await
             .unwrap();
         assert!(blocks(&sets, 1).await);
@@ -716,7 +722,7 @@ mod tests {
         assert!(!blocks(&sets, 3).await);
 
         // No watermark → live tier: all three are members.
-        let sets = fresh_tier_block_list(&sources, None, None, None)
+        let sets = fresh_tier_block_list(&sources, None, None, None, None)
             .await
             .unwrap();
         for id in [1, 2, 3] {
@@ -750,7 +756,7 @@ mod tests {
         )]
         .into_iter()
         .collect();
-        let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks))
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks))
             .await
             .unwrap();
         assert!(blocks(&sets, 1).await); // gen 1, whole
@@ -760,32 +766,32 @@ mod tests {
         assert!(!blocks(&sets, 100).await); // gen 3 — after the snapshot
     }
 
-    /// A flushed generation at or above the active generation was produced by a
+    /// An SSTable at or above the active generation was produced by a
     /// flush after the snapshot and is excluded; one strictly below it is
     /// immutable and included.
     #[tokio::test]
-    async fn fresh_tier_watermark_excludes_flushed_at_or_above_active() {
+    async fn fresh_tier_watermark_excludes_sstables_at_or_above_active() {
         use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark;
         use crate::dataset::{Dataset, WriteParams};
         use arrow_array::RecordBatchIterator;
         use std::collections::HashMap;
 
-        // A flushed generation 2 holding pk=5, staged as a flushed dataset with
+        // An SSTable 2 holding pk=5, staged as an SSTable dataset with
         // its standalone PK sidecar (what the on-disk membership probes).
-        let flushed_batch = id_batch(&[5]);
-        let schema = flushed_batch.schema();
+        let sstable_batch = id_batch(&[5]);
+        let schema = sstable_batch.schema();
         let tmp = tempfile::tempdir().unwrap();
         let path = format!("{}/gen2", tmp.path().to_str().unwrap());
-        let reader = RecordBatchIterator::new(vec![Ok(flushed_batch.clone())], schema.clone());
+        let reader = RecordBatchIterator::new(vec![Ok(sstable_batch.clone())], schema.clone());
         Dataset::write(reader, &path, Some(WriteParams::default()))
             .await
             .unwrap();
-        write_pk_sidecar(&path, &[flushed_batch], &["id"])
+        write_pk_sidecar(&path, &[sstable_batch], &["id"])
             .await
             .unwrap();
 
         let shard = Uuid::new_v4();
-        let sources = vec![LsmDataSource::FlushedMemTable {
+        let sources = vec![LsmDataSource::SsTable {
             path,
             shard_id: shard,
             generation: LsmGeneration::memtable(2),
@@ -801,7 +807,7 @@ mod tests {
         )]
         .into_iter()
         .collect();
-        let sets = fresh_tier_block_list(&sources, None, None, Some(&at))
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&at))
             .await
             .unwrap();
         assert!(!blocks(&sets, 5).await);
@@ -816,7 +822,7 @@ mod tests {
         )]
         .into_iter()
         .collect();
-        let sets = fresh_tier_block_list(&sources, None, None, Some(&above))
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&above))
             .await
             .unwrap();
         assert!(blocks(&sets, 5).await);
diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
index 2947ef1464f..9d01687e595 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
@@ -25,12 +25,14 @@ use uuid::Uuid;
 
 use super::collector::{InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector};
 use super::data_source::{FreshTierWatermark, ShardSnapshot};
-use super::flushed_cache::{DatasetCache, GenerationWarmer};
 use super::planner::LsmScanPlanner;
 use super::point_lookup::LsmPointLookupPlanner;
 use super::projection::validate_projection_names;
+use super::sstable_cache::{DatasetCache, SsTableWarmer};
 use crate::dataset::Dataset;
+use crate::dataset::mem_wal::util::derived_store_params;
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Vector (KNN) search state, set by [`LsmScanner::nearest`] and friends. Mirrors
 /// the subset of `lance::dataset::scanner::Query` the LSM vector planner honors.
@@ -90,7 +92,7 @@ fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option<Vec<ScalarValue>
 }
 
 /// Either a base Lance table, or an explicit base path used to resolve
-/// flushed-generation directories when no base dataset is configured.
+/// SSTable directories when no base dataset is configured.
 enum BaseSource {
     Table(Arc<Dataset>),
     PathOnly(String),
@@ -150,12 +152,12 @@ fn key_to_fsl(key: &dyn Array, dim: i32) -> Result<FixedSizeListArray> {
     Ok(builder.finish())
 }
 
-/// Scanner for LSM tree data spanning base table, flushed MemTables, and active MemTable.
+/// Scanner for LSM tree data spanning base table, SSTables, and active MemTable.
 ///
 /// This scanner provides a unified interface for querying data across multiple
 /// LSM tree levels:
-/// - Base table (merged data, generation = 0)
-/// - Flushed MemTables (persisted but not yet merged, generation = 1, 2, ...)
+/// - Base table (compacted data, generation = 0)
+/// - SSTables (persisted but not yet compacted, generation = 1, 2, ...)
 /// - Active MemTable (in-memory buffer, highest generation)
 ///
 /// The scanner automatically handles deduplication by primary key, keeping
@@ -206,15 +208,17 @@ pub struct LsmScanner {
     // Primary key columns (required for deduplication)
     pk_columns: Vec<String>,
 
-    /// Session threaded into flushed-generation opens so the first open of
-    /// each generation populates the shared index / file-metadata caches.
-    /// Defaults to the base table's session when one is present.
+    /// Session for opening SSTables (shares the base's caches).
+    /// Defaults to the base table's session.
     session: Option<Arc<Session>>,
-    /// Cache of opened flushed-generation datasets. When set, repeated
+    /// Store params for opening SSTables, reusing the base dataset's
+    /// store. Defaults to the base table's params.
+    store_params: Option<ObjectStoreParams>,
+    /// Cache of opened SSTable datasets. When set, repeated
     /// queries against the same generation skip the manifest read entirely.
-    flushed_cache: Option<Arc<dyn DatasetCache>>,
-    /// Optional warmer fired on first open of a flushed generation.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
+    sstable_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of an SSTable.
+    warmer: Option<Arc<dyn SsTableWarmer>>,
     /// Over-fetch multiple for block-listed sources in search plans
     /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]).
     overfetch_factor: Option<f64>,
@@ -225,7 +229,7 @@ impl LsmScanner {
     ///
     /// # Arguments
     ///
-    /// * `base_table` - The base Lance table (merged data)
+    /// * `base_table` - The base Lance table (compacted data)
     /// * `shard_snapshots` - Snapshots of shard states from MemWAL index
     /// * `pk_columns` - Primary key column names for deduplication
     pub fn new(
@@ -239,6 +243,10 @@ impl LsmScanner {
         // the shared index / metadata caches without extra wiring. An
         // explicit `with_session` still overrides this.
         let session = Some(base_table.session());
+        // The scanner only ever opens SSTables with these — the base
+        // table is already open and handed in — so they must not carry a
+        // path-bound store binding.
+        let store_params = base_table.store_params().map(derived_store_params);
         Self {
             base: BaseSource::Table(base_table),
             schema: Arc::new(arrow_schema),
@@ -254,25 +262,26 @@ impl LsmScanner {
             with_memtable_gen: false,
             pk_columns,
             session,
-            flushed_cache: None,
+            store_params,
+            sstable_cache: None,
             warmer: None,
             overfetch_factor: None,
         }
     }
 
     /// Create a scanner that reads only the fresh tier (active memtable and
-    /// flushed generations) without including a base Lance table.
+    /// SSTables) without including a base Lance table.
     ///
     /// This is useful when the caller owns the base read path separately and
-    /// only needs the WAL's contribution: active memtable ∪ L0 flushed
-    /// generations. Deduplication semantics are unchanged — newer generations
+    /// only needs the WAL's contribution: active memtable ∪ L0 SSTables.
+    /// Deduplication semantics are unchanged — newer generations
     /// still win on PK conflicts.
     ///
     /// # Arguments
     ///
     /// * `schema` - Schema used for projection, filter parsing, and empty plans.
-    ///   Should match the schema flushed generations were written with.
-    /// * `base_path` - Table-root URI used to resolve relative flushed paths.
+    ///   Should match the schema SSTables were written with.
+    /// * `base_path` - Table-root URI used to resolve relative SSTable paths.
     /// * `shard_snapshots` - Snapshots of shard states from MemWAL index.
     /// * `pk_columns` - Primary key column names for deduplication.
     pub fn without_base_table(
@@ -296,7 +305,8 @@ impl LsmScanner {
             with_memtable_gen: false,
             pk_columns,
             session: None,
-            flushed_cache: None,
+            store_params: None,
+            sstable_cache: None,
             warmer: None,
             overfetch_factor: None,
         }
@@ -331,33 +341,40 @@ impl LsmScanner {
         self
     }
 
-    /// Thread an existing session into flushed-generation opens.
-    ///
-    /// The first open of each flushed generation then populates the shared
-    /// index / file-metadata caches, so later queries skip re-decoding them.
-    /// When a base table is configured this defaults to its session; call
-    /// this to override (e.g. on a fresh-tier-only scanner that owns its own
-    /// long-lived session).
+    /// Set the session used to open SSTables. Defaults to the base
+    /// table's; set explicitly on a fresh-tier-only scanner (no base table).
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
-    /// Inject a cache of opened flushed-generation datasets.
+    /// Set the store params used to open SSTables. Defaults to the
+    /// base table's; set explicitly on a fresh-tier-only scanner (no base table).
+    ///
+    /// Pass the params the *base* was opened with. As in [`Self::new`], they are
+    /// adapted for generation URIs: a path-bound `object_store` binding would
+    /// redirect every generation open at the base table itself, so it is dropped
+    /// while storage options, wrapper, and credentials carry over.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(derived_store_params(&store_params));
+        self
+    }
+
+    /// Inject a cache of opened SSTable datasets.
     ///
     /// With a cache, repeated queries against the same generation become a
     /// pure `Arc::clone` with no manifest read or object-store I/O. The cache
     /// is owned and sized by the caller (any [`DatasetCache`] impl, e.g.
-    /// [`FlushedMemTableCache`](super::FlushedMemTableCache)); not set by
+    /// [`SsTableCache`](super::SsTableCache)); not set by
     /// default, so behavior is unchanged unless opted in.
-    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
-        self.flushed_cache = Some(cache);
+    pub fn with_sstable_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
+        self.sstable_cache = Some(cache);
         self
     }
 
-    /// Inject the warmer fired on first open of a flushed generation. Not set by
+    /// Inject the warmer fired on first open of an SSTable. Not set by
     /// default, so behavior is unchanged unless opted in.
-    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+    pub fn with_warmer(mut self, warmer: Arc<dyn SsTableWarmer>) -> Self {
         self.warmer = Some(warmer);
         self
     }
@@ -419,7 +436,7 @@ impl LsmScanner {
     }
 
     /// Find the `k` nearest neighbors of `key` in `column`. Routes `create_plan`
-    /// through the LSM vector planner (base ∪ flushed ∪ in-memory). Mirrors
+    /// through the LSM vector planner (base ∪ SSTables ∪ in-memory). Mirrors
     /// [`crate::dataset::scanner::Scanner::nearest`]; the LSM path supports a
     /// single Float32 query vector. When combined with an offset, the LSM path
     /// fetches `k + offset` per source before applying the final page. Tune with
@@ -537,7 +554,7 @@ impl LsmScanner {
         Arc::new(GlobalLimitExec::new(plan, skip, self.limit))
     }
 
-    /// Vector (KNN) search across base ∪ flushed ∪ in-memory, via the LSM vector
+    /// Vector (KNN) search across base ∪ SSTables ∪ in-memory, via the LSM vector
     /// planner. Honors the builder filter as a prefilter.
     async fn plan_vector(&self) -> Result<Arc<dyn ExecutionPlan>> {
         let nearest = self
@@ -564,8 +581,11 @@ impl LsmScanner {
         if let Some(session) = &self.session {
             planner = planner.with_session(session.clone());
         }
-        if let Some(cache) = &self.flushed_cache {
-            planner = planner.with_flushed_cache(cache.clone());
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
+        if let Some(cache) = &self.sstable_cache {
+            planner = planner.with_sstable_cache(cache.clone());
         }
         if let Some(warmer) = &self.warmer {
             planner = planner.with_warmer(warmer.clone());
@@ -588,7 +608,7 @@ impl LsmScanner {
         Ok(self.apply_limit_offset(plan))
     }
 
-    /// Full-text search across base ∪ flushed ∪ in-memory, via the LSM FTS
+    /// Full-text search across base ∪ SSTables ∪ in-memory, via the LSM FTS
     /// planner. Query/scanner limits bound per-source fetches when present;
     /// otherwise the search remains unbounded and any offset is applied above.
     async fn plan_fts(&self) -> Result<Arc<dyn ExecutionPlan>> {
@@ -640,8 +660,11 @@ impl LsmScanner {
         if let Some(session) = &self.session {
             planner = planner.with_session(session.clone());
         }
-        if let Some(cache) = &self.flushed_cache {
-            planner = planner.with_flushed_cache(cache.clone());
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
+        if let Some(cache) = &self.sstable_cache {
+            planner = planner.with_sstable_cache(cache.clone());
         }
         if let Some(warmer) = &self.warmer {
             planner = planner.with_warmer(warmer.clone());
@@ -660,7 +683,7 @@ impl LsmScanner {
         Ok(self.apply_limit_offset(plan))
     }
 
-    /// Plain (filter / projection / limit) scan over base ∪ flushed ∪ in-memory.
+    /// Plain (filter / projection / limit) scan over base ∪ SSTables ∪ in-memory.
     async fn plan_scan(&self) -> Result<Arc<dyn ExecutionPlan>> {
         let collector = self.build_collector();
         let base_schema = self.schema();
@@ -685,8 +708,11 @@ impl LsmScanner {
             if let Some(session) = &self.session {
                 planner = planner.with_session(session.clone());
             }
-            if let Some(cache) = &self.flushed_cache {
-                planner = planner.with_flushed_cache(cache.clone());
+            if let Some(store_params) = &self.store_params {
+                planner = planner.with_store_params(store_params.clone());
+            }
+            if let Some(cache) = &self.sstable_cache {
+                planner = planner.with_sstable_cache(cache.clone());
             }
             if let Some(warmer) = &self.warmer {
                 planner = planner.with_warmer(warmer.clone());
@@ -704,15 +730,15 @@ impl LsmScanner {
         if let Some(session) = &self.session {
             planner = planner.with_session(session.clone());
         }
-        if let Some(cache) = &self.flushed_cache {
-            planner = planner.with_flushed_cache(cache.clone());
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
+        if let Some(cache) = &self.sstable_cache {
+            planner = planner.with_sstable_cache(cache.clone());
         }
         if let Some(warmer) = &self.warmer {
             planner = planner.with_warmer(warmer.clone());
         }
-        if let Some(factor) = self.overfetch_factor {
-            planner = planner.with_overfetch_factor(factor);
-        }
 
         planner
             .plan_scan(
@@ -727,7 +753,7 @@ impl LsmScanner {
     }
 
     /// Find rows matching a full-text query. Routes `create_plan` through the
-    /// LSM FTS planner (base ∪ flushed ∪ in-memory), local-scored by BM25 and
+    /// LSM FTS planner (base ∪ SSTables ∪ in-memory), local-scored by BM25 and
     /// merged by `_score` DESC. Mirrors
     /// [`crate::dataset::scanner::Scanner::full_text_search`]: the searched
     /// column(s) come from the query (set via `FullTextSearchQuery::with_column`);
@@ -778,7 +804,7 @@ impl LsmScanner {
     }
 
     /// Test which `pks` have been (re)written in the WAL fresh tier — the active
-    /// and frozen memtables and flushed generations this scanner spans — i.e.
+    /// and frozen memtables and SSTables this scanner spans — i.e.
     /// are shadowed above the base table. `pks` is a batch whose columns include
     /// the primary-key columns; the returned `Vec<bool>` is aligned with its
     /// rows. Hashing matches the scanner's internal dedup, so the caller never
@@ -802,7 +828,8 @@ impl LsmScanner {
         let memberships = super::block_list::fresh_tier_block_list(
             &sources,
             self.session.as_ref(),
-            self.flushed_cache.as_ref(),
+            self.store_params.as_ref(),
+            self.sstable_cache.as_ref(),
             watermarks,
         )
         .await?;
@@ -915,13 +942,13 @@ mod tests {
         let snapshot = ShardSnapshot::new(shard_id)
             .with_spec_id(1)
             .with_current_generation(5)
-            .with_flushed_generation(1, "path/gen_1".to_string())
-            .with_flushed_generation(2, "path/gen_2".to_string());
+            .with_sstable(1, "path/gen_1".to_string())
+            .with_sstable(2, "path/gen_2".to_string());
 
         assert_eq!(snapshot.shard_id, shard_id);
         assert_eq!(snapshot.spec_id, 1);
         assert_eq!(snapshot.current_generation, 5);
-        assert_eq!(snapshot.flushed_generations.len(), 2);
+        assert_eq!(snapshot.sstables.len(), 2);
     }
 
     #[test]
@@ -973,44 +1000,69 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn invalid_overfetch_factor_is_rejected() {
-        let shard = Uuid::new_v4();
-        let scanner = LsmScanner::without_base_table(
+    async fn overfetch_factor_only_applies_to_searches() {
+        // Plain scans refill exactly via LocalLimitExec, so overfetch_factor is
+        // search-only and must not affect scan planning.
+        let scan = LsmScanner::without_base_table(
             pk_schema(),
-            "memory://t",
+            "memory://scan",
             vec![],
             vec!["id".to_string()],
         )
-        .with_in_memory_memtables(
-            shard,
-            InMemoryMemTables {
-                active: mk_pk_memtable(&[1, 2], 2),
-                frozen: vec![],
-            },
+        .with_overfetch_factor(0.5)
+        .try_into_batch()
+        .await
+        .unwrap();
+        assert_eq!(scan.num_rows(), 0);
+
+        let vector_schema = pk_schema_with(arrow_schema::Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)),
+                4,
+            ),
+            false,
+        ));
+        let vector_search = LsmScanner::without_base_table(
+            vector_schema,
+            "memory://vector",
+            vec![],
+            vec!["id".to_string()],
         )
+        .nearest(
+            "vector",
+            &arrow_array::Float32Array::from(vec![0.0f32, 0.0, 0.0, 0.0]),
+            1,
+        )
+        .unwrap()
         .with_overfetch_factor(0.5);
-
-        let Err(err) = scanner.try_into_stream().await else {
-            panic!("invalid overfetch factor should fail planning");
+        let Err(err) = vector_search.try_into_stream().await else {
+            panic!("invalid overfetch factor should fail vector search planning");
         };
         assert!(
             err.to_string().contains("overfetch_factor"),
             "unexpected error for invalid overfetch factor: {err}"
         );
 
-        let empty_scanner = LsmScanner::without_base_table(
-            pk_schema(),
-            "memory://empty",
+        let fts_search = LsmScanner::without_base_table(
+            pk_schema_with(arrow_schema::Field::new("text", DataType::Utf8, true)),
+            "memory://fts",
             vec![],
             vec!["id".to_string()],
         )
+        .full_text_search(
+            FullTextSearchQuery::new("lance".to_string())
+                .with_column("text".to_string())
+                .unwrap(),
+        )
+        .unwrap()
         .with_overfetch_factor(0.5);
-        let Err(err) = empty_scanner.try_into_stream().await else {
-            panic!("invalid overfetch factor should fail even when there are no sources");
+        let Err(err) = fts_search.try_into_stream().await else {
+            panic!("invalid overfetch factor should fail full-text search planning");
         };
         assert!(
             err.to_string().contains("overfetch_factor"),
-            "unexpected error for invalid empty-source overfetch factor: {err}"
+            "unexpected error for invalid overfetch factor: {err}"
         );
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
index 6645f159b12..b8df6edcb2e 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
@@ -49,18 +49,18 @@ pub struct InMemoryMemTables {
 ///
 /// This collector gathers all data sources that need to be scanned
 /// for a query, including:
-/// - The base table (merged data) — optional; omit for fresh-tier-only scans
-/// - Flushed MemTables from each shard
+/// - The base table (compacted data) — optional; omit for fresh-tier-only scans
+/// - SSTables from each shard
 /// - In-memory memtables per shard (active + frozen-awaiting-flush)
 ///
 /// When the base table is omitted (see [`Self::without_base_table`]), `collect`
-/// returns only flushed-generation and active-memtable sources. This is used
+/// returns only SSTable and active-memtable sources. This is used
 /// by callers that own the base read path elsewhere and only need the WAL's
-/// fresh tier (active memtable ∪ L0 flushed generations).
+/// fresh tier (active memtable ∪ L0 SSTables).
 pub struct LsmDataSourceCollector {
     /// Base Lance table (None when scanning only the fresh tier).
     base_table: Option<Arc<Dataset>>,
-    /// Base path for resolving relative flushed-generation paths.
+    /// Base path for resolving relative SSTable paths.
     base_path: String,
     /// Shard snapshots from MemWAL index.
     shard_snapshots: Vec<ShardSnapshot>,
@@ -73,7 +73,7 @@ impl LsmDataSourceCollector {
     ///
     /// # Arguments
     ///
-    /// * `base_table` - The base Lance table (merged data)
+    /// * `base_table` - The base Lance table (compacted data)
     /// * `shard_snapshots` - Snapshots of shard states from MemWAL index
     pub fn new(base_table: Arc<Dataset>, shard_snapshots: Vec<ShardSnapshot>) -> Self {
         // Use the dataset's URI as base path for resolving relative paths.
@@ -89,8 +89,8 @@ impl LsmDataSourceCollector {
 
     /// Create a collector without a base table (fresh-tier scan only).
     ///
-    /// The collector emits only flushed-generation and active-memtable sources.
-    /// `base_path` is the table-root URI used to resolve relative flushed paths
+    /// The collector emits only SSTable and active-memtable sources.
+    /// `base_path` is the table-root URI used to resolve relative SSTable paths
     /// (typically the same URI that would have been the base dataset's URI).
     pub fn without_base_table(
         base_path: impl Into<String>,
@@ -147,16 +147,12 @@ impl LsmDataSourceCollector {
         &self.in_memory_memtables
     }
 
-    /// Whether the collector has any on-disk source (base table or a flushed
-    /// generation). The point-lookup fast path uses this to decide, after
+    /// Whether the collector has any on-disk source (base table or an
+    /// SSTable). The point-lookup fast path uses this to decide, after
     /// missing every in-memory memtable, between "definitely absent" (`false`)
     /// and "must consult disk via the plan path" (`true`). Cheap: no allocation.
     pub fn has_on_disk_sources(&self) -> bool {
-        self.base_table.is_some()
-            || self
-                .shard_snapshots
-                .iter()
-                .any(|s| !s.flushed_generations.is_empty())
+        self.base_table.is_some() || self.shard_snapshots.iter().any(|s| !s.sstables.is_empty())
     }
 
     /// The in-memory memtables (active + frozen across all shards) as
@@ -233,10 +229,10 @@ impl LsmDataSourceCollector {
     /// frozen memtable. During the post-flush grace window a generation is both
     /// committed to the manifest (a flushed source) and held in memory (an
     /// in-memory source); it must be served only from memory — which preserves
-    /// the per-batch boundaries the flushed dataset has lost, so as-of reads
+    /// the per-batch boundaries the SSTable dataset has lost, so as-of reads
     /// stay snapshot-bounded — and its on-disk copy skipped to avoid scanning
     /// the generation twice. See `ShardWriterConfig::frozen_memtable_grace`.
-    fn flushed_gen_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool {
+    fn sstable_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool {
         self.in_memory_memtables
             .get(shard_id)
             .is_some_and(|mems| mems.frozen.iter().any(|f| f.generation == generation))
@@ -246,7 +242,7 @@ impl LsmDataSourceCollector {
     ///
     /// Returns sources in a consistent order:
     /// 1. Base table (gen=0), if configured
-    /// 2. Flushed MemTables per shard, ordered by generation
+    /// 2. SSTables per shard, ordered by generation
     /// 3. In-memory memtables per shard (active + frozen-awaiting-flush)
     pub fn collect(&self) -> Result<Vec<LsmDataSource>> {
         let mut sources = Vec::new();
@@ -258,15 +254,15 @@ impl LsmDataSourceCollector {
         }
 
         for snapshot in &self.shard_snapshots {
-            for flushed in &snapshot.flushed_generations {
-                if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) {
+            for sstable in &snapshot.sstables {
+                if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) {
                     continue;
                 }
-                let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
-                sources.push(LsmDataSource::FlushedMemTable {
+                let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path);
+                sources.push(LsmDataSource::SsTable {
                     path,
                     shard_id: snapshot.shard_id,
-                    generation: LsmGeneration::memtable(flushed.generation),
+                    generation: LsmGeneration::memtable(sstable.generation),
                 });
             }
         }
@@ -299,15 +295,15 @@ impl LsmDataSourceCollector {
                 continue;
             }
 
-            for flushed in &snapshot.flushed_generations {
-                if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) {
+            for sstable in &snapshot.sstables {
+                if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) {
                     continue;
                 }
-                let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
-                sources.push(LsmDataSource::FlushedMemTable {
+                let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path);
+                sources.push(LsmDataSource::SsTable {
                     path,
                     shard_id: snapshot.shard_id,
-                    generation: LsmGeneration::memtable(flushed.generation),
+                    generation: LsmGeneration::memtable(sstable.generation),
                 });
             }
         }
@@ -325,25 +321,21 @@ impl LsmDataSourceCollector {
 
     /// Get the total number of data sources.
     pub fn num_sources(&self) -> usize {
-        let flushed_count: usize = self
-            .shard_snapshots
-            .iter()
-            .map(|s| s.flushed_generations.len())
-            .sum();
+        let sstable_count: usize = self.shard_snapshots.iter().map(|s| s.sstables.len()).sum();
         let base_count = if self.base_table.is_some() { 1 } else { 0 };
         let in_memory_count: usize = self
             .in_memory_memtables
             .values()
             .map(|m| 1 + m.frozen.len())
             .sum();
-        base_count + flushed_count + in_memory_count
+        base_count + sstable_count + in_memory_count
     }
 
-    /// Resolve a flushed MemTable path to an absolute path.
+    /// Resolve an SSTable path to an absolute path.
     ///
-    /// Flushed MemTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}`
-    /// The `folder_name` is what's stored in `FlushedGeneration.path`.
-    fn resolve_flushed_path(&self, shard_id: &Uuid, folder_name: &str) -> String {
+    /// SSTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}`
+    /// The `folder_name` is what's stored in `SsTable.path`.
+    fn resolve_sstable_path(&self, shard_id: &Uuid, folder_name: &str) -> String {
         format!("{}/_mem_wal/{}/{}", self.base_path, shard_id, folder_name)
     }
 }
@@ -351,7 +343,7 @@ impl LsmDataSourceCollector {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::dataset::mem_wal::scanner::data_source::FlushedGeneration;
+    use crate::dataset::mem_wal::scanner::data_source::SsTable;
 
     fn create_test_snapshots() -> Vec<ShardSnapshot> {
         let shard_a = Uuid::new_v4();
@@ -362,12 +354,12 @@ mod tests {
                 shard_id: shard_a,
                 spec_id: 1,
                 current_generation: 3,
-                flushed_generations: vec![
-                    FlushedGeneration {
+                sstables: vec![
+                    SsTable {
                         generation: 1,
                         path: "abc_gen_1".to_string(),
                     },
-                    FlushedGeneration {
+                    SsTable {
                         generation: 2,
                         path: "def_gen_2".to_string(),
                     },
@@ -377,7 +369,7 @@ mod tests {
                 shard_id: shard_b,
                 spec_id: 1,
                 current_generation: 2,
-                flushed_generations: vec![FlushedGeneration {
+                sstables: vec![SsTable {
                     generation: 1,
                     path: "xyz_gen_1".to_string(),
                 }],
@@ -390,8 +382,8 @@ mod tests {
         let snapshots = create_test_snapshots();
         // 1 base table + 2 flushed from shard_a + 1 flushed from shard_b = 4
         // Using a mock dataset is complex, so we just test the counting logic
-        assert_eq!(snapshots[0].flushed_generations.len(), 2);
-        assert_eq!(snapshots[1].flushed_generations.len(), 1);
+        assert_eq!(snapshots[0].sstables.len(), 2);
+        assert_eq!(snapshots[1].sstables.len(), 1);
     }
 
     #[test]
@@ -466,10 +458,10 @@ mod tests {
     /// During the post-flush grace window a generation is both committed to the
     /// manifest (a flushed source) and still pinned in memory (a frozen
     /// source). The collector must emit it once, from memory — so as-of reads
-    /// keep batch-resolved membership — and skip the on-disk copy. Flushed
-    /// generations NOT pinned in memory are still emitted from disk.
+    /// keep batch-resolved membership — and skip the on-disk copy. SSTables
+    /// NOT pinned in memory are still emitted from disk.
     #[test]
-    fn test_collect_suppresses_flushed_gen_pinned_in_memory() {
+    fn test_collect_suppresses_sstable_pinned_in_memory() {
         let shard = Uuid::new_v4();
         // Manifest lists gens 1 and 2 as flushed; gen 2 is still pinned in
         // memory (just flushed, within grace), gen 1 has been swept.
@@ -477,12 +469,12 @@ mod tests {
             shard_id: shard,
             spec_id: 0,
             current_generation: 3,
-            flushed_generations: vec![
-                FlushedGeneration {
+            sstables: vec![
+                SsTable {
                     generation: 1,
                     path: "gen_1".to_string(),
                 },
-                FlushedGeneration {
+                SsTable {
                     generation: 2,
                     path: "gen_2".to_string(),
                 },
@@ -498,7 +490,7 @@ mod tests {
         let sources = collector.collect().unwrap();
         // gen 1: on-disk (not pinned). gen 2: in-memory only (pinned, disk
         // copy suppressed). gen 3: active. No duplicate gen 2.
-        let flushed: Vec<u64> = sources
+        let sstable_gens: Vec<u64> = sources
             .iter()
             .filter(|s| !s.is_active_memtable())
             .map(|s| s.generation().as_u64())
@@ -508,7 +500,11 @@ mod tests {
             .filter(|s| s.is_active_memtable())
             .map(|s| s.generation().as_u64())
             .collect();
-        assert_eq!(flushed, vec![1], "only the unpinned flushed gen from disk");
+        assert_eq!(
+            sstable_gens,
+            vec![1],
+            "only the unpinned SSTable gen from disk"
+        );
         assert_eq!(in_memory, vec![2, 3], "pinned gen 2 served from memory");
     }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
index 0d5f3fdc925..86e6041142b 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
@@ -44,7 +44,7 @@ pub struct FreshTierWatermark {
 pub struct LsmGeneration(u64);
 
 impl LsmGeneration {
-    /// Generation for the base table (merged data).
+    /// Generation for the base table (compacted data).
     pub const BASE_TABLE: Self = Self(0);
 
     /// Create a generation for a MemTable.
@@ -93,12 +93,12 @@ impl Default for LsmGeneration {
     }
 }
 
-/// A flushed generation with its storage path.
+/// An SSTable with its storage path.
 #[derive(Debug, Clone)]
-pub struct FlushedGeneration {
+pub struct SsTable {
     /// Generation number.
     pub generation: u64,
-    /// Path to the flushed MemTable directory (relative to table root).
+    /// Path to the SSTable directory (relative to table root).
     pub path: String,
 }
 
@@ -114,8 +114,8 @@ pub struct ShardSnapshot {
     pub spec_id: u32,
     /// Current generation being written (next flush will be this generation).
     pub current_generation: u64,
-    /// List of flushed generations and their paths.
-    pub flushed_generations: Vec<FlushedGeneration>,
+    /// List of SSTables and their paths.
+    pub sstables: Vec<SsTable>,
 }
 
 impl ShardSnapshot {
@@ -125,7 +125,7 @@ impl ShardSnapshot {
             shard_id,
             spec_id: 0,
             current_generation: 1,
-            flushed_generations: Vec::new(),
+            sstables: Vec::new(),
         }
     }
 
@@ -141,10 +141,9 @@ impl ShardSnapshot {
         self
     }
 
-    /// Add a flushed generation.
-    pub fn with_flushed_generation(mut self, generation: u64, path: String) -> Self {
-        self.flushed_generations
-            .push(FlushedGeneration { generation, path });
+    /// Add an SSTable.
+    pub fn with_sstable(mut self, generation: u64, path: String) -> Self {
+        self.sstables.push(SsTable { generation, path });
         self
     }
 }
@@ -156,9 +155,9 @@ pub enum LsmDataSource {
         /// The base dataset.
         dataset: Arc<Dataset>,
     },
-    /// Flushed MemTable stored as Lance table on disk.
-    FlushedMemTable {
-        /// Absolute path to the flushed MemTable directory.
+    /// SSTable stored as Lance table on disk.
+    SsTable {
+        /// Absolute path to the SSTable directory.
         path: String,
         /// Shard this MemTable belongs to.
         shard_id: Uuid,
@@ -185,7 +184,7 @@ impl LsmDataSource {
     pub fn generation(&self) -> LsmGeneration {
         match self {
             Self::BaseTable { .. } => LsmGeneration::BASE_TABLE,
-            Self::FlushedMemTable { generation, .. } => *generation,
+            Self::SsTable { generation, .. } => *generation,
             Self::ActiveMemTable { generation, .. } => *generation,
         }
     }
@@ -194,7 +193,7 @@ impl LsmDataSource {
     pub fn shard_id(&self) -> Option<Uuid> {
         match self {
             Self::BaseTable { .. } => None,
-            Self::FlushedMemTable { shard_id, .. } => Some(*shard_id),
+            Self::SsTable { shard_id, .. } => Some(*shard_id),
             Self::ActiveMemTable { shard_id, .. } => Some(*shard_id),
         }
     }
@@ -213,7 +212,7 @@ impl LsmDataSource {
     pub fn display_name(&self) -> String {
         match self {
             Self::BaseTable { .. } => "base_table".to_string(),
-            Self::FlushedMemTable {
+            Self::SsTable {
                 shard_id,
                 generation,
                 ..
@@ -279,14 +278,14 @@ mod tests {
         let snapshot = ShardSnapshot::new(shard_id)
             .with_spec_id(1)
             .with_current_generation(5)
-            .with_flushed_generation(1, "abc123_gen_1".to_string())
-            .with_flushed_generation(2, "def456_gen_2".to_string());
+            .with_sstable(1, "abc123_gen_1".to_string())
+            .with_sstable(2, "def456_gen_2".to_string());
 
         assert_eq!(snapshot.shard_id, shard_id);
         assert_eq!(snapshot.spec_id, 1);
         assert_eq!(snapshot.current_generation, 5);
-        assert_eq!(snapshot.flushed_generations.len(), 2);
-        assert_eq!(snapshot.flushed_generations[0].generation, 1);
-        assert_eq!(snapshot.flushed_generations[1].generation, 2);
+        assert_eq!(snapshot.sstables.len(), 2);
+        assert_eq!(snapshot.sstables[0].generation, 1);
+        assert_eq!(snapshot.sstables[1].generation, 2);
     }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
index 632b08a753f..e3a710bd367 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
@@ -5,7 +5,6 @@
 //!
 //! Used in point lookup queries to skip generations that definitely don't contain the key.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -134,10 +133,6 @@ impl ExecutionPlan for BloomFilterGuardExec {
         "BloomFilterGuardExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
index 9e158c86b4a..c212b47c013 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
@@ -5,7 +5,6 @@
 //!
 //! Used in point lookup queries to stop searching after finding the first match.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -111,10 +110,6 @@ impl ExecutionPlan for CoalesceFirstExec {
         "CoalesceFirstExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
index ba9d565316f..9c1d0060a78 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
@@ -3,7 +3,6 @@
 
 //! MemTable generation tagging execution node.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -100,10 +99,6 @@ impl ExecutionPlan for MemtableGenTagExec {
         "MemtableGenTagExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
index 89dbd7adc61..ef2dc695dfa 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
@@ -5,7 +5,7 @@
 //!
 //! Drops a row when any newer generation's membership ([`GenMembership`])
 //! contains its primary key — in-memory generations probe their PK index by
-//! value, flushed generations probe their on-disk PK BTree. Each generation is
+//! value, SSTables probe their on-disk PK BTree. Each generation is
 //! probed once per batch (see the perf note below). Used both as the KNN
 //! post-filter (vector search, with over-fetch) and the cross-generation scan
 //! filter (`k = 0`).
@@ -22,12 +22,11 @@
 //! `BTreeIndex::contains_keys` (one page pass, no per-key `SearchResult`
 //! allocation); the in-memory arm maps a sync PK lookup over the keys. Probes
 //! are not disk-bound in steady state: the opened index and its (small,
-//! memtable-sized) pages are held by the injected `FlushedMemTableCache` /
+//! memtable-sized) pages are held by the injected `SsTableCache` /
 //! `LanceCache`, so after the first touch every probe is memory-resident.
 //! Already-blocked rows are dropped from the key set before probing older
 //! generations, preserving the per-row short-circuit.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -109,10 +108,6 @@ impl ExecutionPlan for PkBlockFilterExec {
         "PkBlockFilterExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.input.schema()
     }
@@ -185,8 +180,8 @@ struct PkBlockFilterStream {
     warned: bool,
 }
 
-/// Keep only the rows no newer-gen membership contains. Async because flushed
-/// generations are probed against their on-disk PK BTree.
+/// Keep only the rows no newer-gen membership contains. Async because SSTables
+/// are probed against their on-disk PK BTree.
 async fn filter_batch(batch: RecordBatch, config: Arc<FilterConfig>) -> DFResult<RecordBatch> {
     let FilterConfig {
         pk_columns,
@@ -322,7 +317,7 @@ mod tests {
             let (bp, off, _) = store.append(b.clone()).unwrap();
             index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
         }
-        let max_visible_row = store.max_visible_row(index.max_visible_batch_position());
+        let max_visible_row = store.max_visible_row(index.visible_count());
         GenMembership::InMemory {
             index_store: Arc::new(index),
             max_visible_row,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
index d783540bf6b..d876ec39071 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
@@ -4,7 +4,7 @@
 //! Full-text search planner for LSM scanner (local scoring).
 //!
 //! Builds an execution plan that scores an FTS query across the base
-//! table, flushed memtable generations, and the active/frozen-undrained
+//! table, SSTable generations, and the active/frozen-undrained
 //! in-memory memtables, returning rows ordered by BM25 `_score` DESC.
 //!
 //! # Scoring
@@ -22,17 +22,17 @@
 //! benchmark in this PR shows it carries a real latency penalty, so the
 //! local path lands first and the global option is optimized separately.
 //!
-//! Staleness: within a flushed generation, the deletion vector written
+//! Staleness: within an SSTable, the deletion vector written
 //! at flush time (see #6929) already masks rows superseded by a newer
 //! generation, so per-source results are clean within each tier. The
-//! same primary key can still appear across tiers (active vs flushed)
+//! same primary key can still appear across tiers (active vs SSTable)
 //! when an updated row sits in the active memtable while the older
-//! copy lives in a flushed generation; cross-tier deduplication is
+//! copy lives in an SSTable; cross-tier deduplication is
 //! left to the caller in local mode.
 //!
 //! Everything here is contained in the `mem_wal` module — it reuses the
 //! existing per-source FTS read paths (`scanner.full_text_search` for
-//! base/flushed Lance datasets, `MemTableScanner` for the active
+//! base/SSTable Lance datasets, `MemTableScanner` for the active
 //! memtable) and requires no changes to `lance-index`.
 
 use std::sync::Arc;
@@ -54,10 +54,11 @@ use super::block_list::compute_source_block_lists;
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
 use super::exec::PkBlockFilterExec;
-use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{project_to_canonical, validate_projection_names};
+use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable};
 use crate::dataset::mem_wal::memtable::scanner::MemTableScanner;
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// `_score` column name in FTS results — kept aligned with
 /// `lance_index::scalar::inverted::SCORE_COL` so this module doesn't
@@ -97,7 +98,7 @@ fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool {
                 .get_fts_by_column(column)
                 .is_some_and(|index| !index.is_empty())
                 && batch_store
-                    .max_visible_row(index_store.max_visible_batch_position())
+                    .max_visible_row(index_store.visible_count())
                     .is_some()
         }
         _ => false,
@@ -109,16 +110,18 @@ pub struct LsmFtsSearchPlanner {
     collector: LsmDataSourceCollector,
     pk_columns: Vec<String>,
     base_schema: SchemaRef,
-    /// Session threaded into flushed-generation opens (shared caches).
+    /// Session threaded into SSTable opens (shared caches).
     session: Option<Arc<Session>>,
-    /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<dyn DatasetCache>>,
-    /// Optional warmer fired on first open of a flushed generation.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Store params for opening SSTables, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
+    /// Cache of opened SSTable datasets.
+    sstable_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of an SSTable.
+    warmer: Option<Arc<dyn SsTableWarmer>>,
     /// Over-fetch multiple for blocked sources.
     overfetch_factor: f64,
     /// Optional prefilter predicate applied to every source arm so FTS hits
-    /// failing the predicate are dropped. Base/flushed arms use the dataset
+    /// failing the predicate are dropped. Base/SSTable arms use the dataset
     /// scanner's native filter; memtable arms filter the materialized hits.
     filter: Option<Expr>,
 }
@@ -135,7 +138,8 @@ impl LsmFtsSearchPlanner {
             pk_columns,
             base_schema,
             session: None,
-            flushed_cache: None,
+            store_params: None,
+            sstable_cache: None,
             warmer: None,
             overfetch_factor: DEFAULT_OVERFETCH_FACTOR,
             filter: None,
@@ -144,7 +148,7 @@ impl LsmFtsSearchPlanner {
 
     /// Attach an optional prefilter predicate. Every source arm restricts its
     /// FTS hits to rows matching the predicate, matching a normal filtered
-    /// full-text scan over base ∪ flushed ∪ in-memory data.
+    /// full-text scan over base ∪ SSTables ∪ in-memory data.
     pub fn with_filter(mut self, filter: Option<Expr>) -> Self {
         self.filter = filter;
         self
@@ -158,22 +162,27 @@ impl LsmFtsSearchPlanner {
         self
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open SSTables.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
-    /// Inject a cache of opened flushed-generation datasets, making repeated
+    /// Set the store params used to open SSTables.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
+    /// Inject a cache of opened SSTable datasets, making repeated
     /// searches against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
-        self.flushed_cache = Some(cache);
+    pub fn with_sstable_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
+        self.sstable_cache = Some(cache);
         self
     }
 
-    /// Inject the warmer fired on first open of a flushed generation.
-    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+    /// Inject the warmer fired on first open of an SSTable.
+    pub fn with_warmer(mut self, warmer: Arc<dyn SsTableWarmer>) -> Self {
         self.warmer = Some(warmer);
         self
     }
@@ -184,7 +193,7 @@ impl LsmFtsSearchPlanner {
     ///
     /// * `column` — text column to search.
     /// * `query` — the FTS query (match / phrase / boolean / fuzzy for
-    ///   base/flushed Lance sources; the active memtable currently
+    ///   base/SSTable Lance sources; the active memtable currently
     ///   supports `MatchQuery`).
     /// * `limit` — optional global top-k to return.
     /// * `projection` — user columns to project. PK columns are
@@ -228,7 +237,8 @@ impl LsmFtsSearchPlanner {
         let block_lists = Box::pin(compute_source_block_lists(
             &sources,
             self.session.as_ref(),
-            self.flushed_cache.as_ref(),
+            self.store_params.as_ref(),
+            self.sstable_cache.as_ref(),
         ))
         .await?;
 
@@ -370,11 +380,12 @@ impl LsmFtsSearchPlanner {
                 scanner.full_text_search(bound_query)?;
                 scanner.create_plan().await
             }
-            LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset = open_flushed_dataset(
+            LsmDataSource::SsTable { path, .. } => {
+                let dataset = open_sstable(
                     path,
                     self.session.as_ref(),
-                    self.flushed_cache.as_ref(),
+                    self.store_params.as_ref(),
+                    self.sstable_cache.as_ref(),
                     self.warmer.as_ref(),
                 )
                 .await?;
@@ -611,7 +622,7 @@ mod tests {
     #[tokio::test]
     async fn local_mode_unions_base_and_active_with_consistent_score_schema() {
         // Regression for the `_score` nullability mismatch between
-        // FtsIndexExec (active arm) and FTS_SCHEMA (base/flushed). The
+        // FtsIndexExec (active arm) and FTS_SCHEMA (base/SSTable). The
         // active-only test below would not catch this — UnionExec rejects
         // schema-inequality, so we need at least one base + one active
         // source to exercise that code path.
@@ -818,12 +829,12 @@ mod tests {
         );
     }
 
-    /// The flushed arm must apply the filter as a true FTS prefilter, and that
+    /// The SSTable arm must apply the filter as a true FTS prefilter, and that
     /// prefiltered candidate set must compose with cross-generation block-list
     /// filtering plus over-fetch. Gen 1's best predicate-matching hit (id=3) is
     /// superseded by gen 2; with over-fetch, gen 1 should still contribute id=4.
     #[tokio::test]
-    async fn prefilter_on_flushed_composes_with_block_list() {
+    async fn prefilter_on_sstable_composes_with_block_list() {
         use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
         use crate::index::DatasetIndexExt;
         use datafusion::prelude::{col, lit};
@@ -837,7 +848,7 @@ mod tests {
 
         // Gen 1: id=1 matches strongly but fails the predicate. id=3 matches
         // strongly but is stale (blocked by gen 2). id=4 is the next live
-        // predicate match that only survives if the flushed arm prefilters and
+        // predicate match that only survives if the SSTable arm prefilters and
         // over-fetches before the block-list drops id=3.
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let mut gen1 = write_dataset(
@@ -875,8 +886,8 @@ mod tests {
 
         let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]);
 
         let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
@@ -890,7 +901,7 @@ mod tests {
                 None,
             )
             .await
-            .expect("planner should produce a filtered flushed plan");
+            .expect("planner should produce a filtered SSTable plan");
 
         let ctx = datafusion::prelude::SessionContext::new();
         let stream = plan.execute(0, ctx.task_ctx()).unwrap();
@@ -911,7 +922,7 @@ mod tests {
         assert_eq!(
             ids,
             vec![4],
-            "flushed FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}"
+            "SSTable FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}"
         );
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
index ec13da1df66..2a83447fb17 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
@@ -18,16 +18,17 @@ use crate::dataset::mem_wal::TOMBSTONE;
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
 use super::exec::{MEMTABLE_GEN_COLUMN, MemtableGenTagExec, PkBlockFilterExec, ROW_ADDRESS_COLUMN};
-use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
     build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical,
     validate_projection_names,
 };
+use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable};
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Combine the user filter (if any) with `NOT _tombstone` so tombstone rows are
 /// dropped from a WAL-arm scan. Used only for sources whose schema carries the
-/// column (active / flushed generations written since deletes existed).
+/// column (active / SSTables written since deletes existed).
 fn fold_not_tombstone(filter: Option<&Expr>) -> Expr {
     let live = !col(TOMBSTONE);
     match filter {
@@ -44,25 +45,14 @@ pub struct LsmScanPlanner {
     pk_columns: Vec<String>,
     /// Schema of the base table.
     base_schema: SchemaRef,
-    /// Session threaded into flushed-generation opens (shared caches).
+    /// Session threaded into SSTable opens (shared caches).
     session: Option<Arc<Session>>,
-    /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<dyn DatasetCache>>,
-    /// Optional warmer fired on first open of a flushed generation.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
-    /// Over-fetch multiple for the per-source limit pushdown: block-listed
-    /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops
-    /// still leave enough live rows. Clamped to `>= 1.0`.
-    ///
-    /// This headroom must also absorb deletes: a tombstone shadows the older
-    /// real row without emitting a replacement (shadow-without-replace), so it
-    /// is pure subtraction from a block-listed source. If delete density inside
-    /// a source's fetch window exceeds `factor - 1`, that source can deliver
-    /// `< k` live rows (a recall shortfall, not wrong content — see the
-    /// under-fetch `warn!` in `PkBlockFilterExec`). Steady-state this is
-    /// self-limiting because L0→base compaction drains tombstones from the
-    /// fresh tier; the exposure is a delete-heavy burst between compactions.
-    overfetch_factor: f64,
+    /// Store params for opening SSTables, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
+    /// Cache of opened SSTable datasets.
+    sstable_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of an SSTable.
+    warmer: Option<Arc<dyn SsTableWarmer>>,
 }
 
 impl LsmScanPlanner {
@@ -77,37 +67,34 @@ impl LsmScanPlanner {
             pk_columns,
             base_schema,
             session: None,
-            flushed_cache: None,
+            store_params: None,
+            sstable_cache: None,
             warmer: None,
-            overfetch_factor: 1.0,
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open SSTables.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
-    /// Inject a cache of opened flushed-generation datasets, making repeated
-    /// queries against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
-        self.flushed_cache = Some(cache);
+    /// Set the store params used to open SSTables.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
         self
     }
 
-    /// Inject the warmer fired on first open of a flushed generation.
-    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
-        self.warmer = Some(warmer);
+    /// Inject a cache of opened SSTable datasets, making repeated
+    /// queries against the same generation a pure `Arc::clone`.
+    pub fn with_sstable_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
+        self.sstable_cache = Some(cache);
         self
     }
 
-    /// Set the over-fetch multiple for the per-source limit pushdown
-    /// (see the field docs). Values below `1.0` are rejected by
-    /// [`Self::plan_scan`].
-    pub fn with_overfetch_factor(mut self, factor: f64) -> Self {
-        self.overfetch_factor = factor;
+    /// Inject the warmer fired on first open of an SSTable.
+    pub fn with_warmer(mut self, warmer: Arc<dyn SsTableWarmer>) -> Self {
+        self.warmer = Some(warmer);
         self
     }
 
@@ -154,7 +141,6 @@ impl LsmScanPlanner {
 
         // 1. Collect all data sources
         let sources = self.collector.collect()?;
-        let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?;
 
         if sources.is_empty() {
             // Return empty plan
@@ -167,7 +153,8 @@ impl LsmScanPlanner {
         let block_lists = Box::pin(super::block_list::compute_source_block_lists(
             &sources,
             self.session.as_ref(),
-            self.flushed_cache.as_ref(),
+            self.store_params.as_ref(),
+            self.sstable_cache.as_ref(),
         ))
         .await?;
 
@@ -177,14 +164,13 @@ impl LsmScanPlanner {
         let sources: Vec<_> = sources.into_iter().rev().collect();
 
         // Per-source limit pushdown: an unordered LIMIT needs only
-        // `offset + limit` live rows from EACH source to fill the global
-        // limit after dedup (any-N semantics), so cap every on-disk source
-        // instead of scanning whole generations and trimming above the
-        // union. Block-listed sources over-fetch by `overfetch_factor` so
-        // cross-gen dedup drops still leave `n_needed` live rows; the
-        // PkBlockFilter warns when that was not enough. The active memtable
-        // is in-memory and within-gen append duplicates are resolved by its
-        // own dedup, so it is never capped here.
+        // `offset + limit` live rows from each source to fill the global limit
+        // (any-N semantics). However, a source with a cross-generation block
+        // list can lose any number of rows after its scan, so a finite fetch
+        // before that filter is not safe. Leave those scans unbounded and let
+        // the LocalLimitExec below pull until it sees `n_needed` live rows or
+        // reaches EOF. Sources without a block filter can still push the limit
+        // down safely. The active memtable is in-memory and is never capped.
         let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0)));
 
         let mut source_plans = Vec::new();
@@ -194,12 +180,9 @@ impl LsmScanPlanner {
             let blocked = block_lists
                 .get(&(source.shard_id(), source.generation()))
                 .cloned();
-            let fetch = match (n_needed, is_active) {
-                (Some(n), false) => Some(if blocked.is_some() && !self.pk_columns.is_empty() {
-                    ((n as f64) * overfetch).ceil() as usize
-                } else {
-                    n
-                }),
+            let has_block_filter = blocked.is_some() && !self.pk_columns.is_empty();
+            let fetch = match (n_needed, is_active, has_block_filter) {
+                (Some(n), false, false) => Some(n),
                 _ => None,
             };
             let scan = self
@@ -207,14 +190,14 @@ impl LsmScanPlanner {
                 .await?;
 
             // Drop cross-generation stale rows (PKs superseded by a newer gen).
-            // With a limit, `k = n_needed` arms the under-fetch warning; with
-            // no limit `k = 0` keeps it silent.
+            // Plain scans refill exactly, so keep the approximate-search
+            // under-fetch warning disabled with k = 0.
             let scan = match blocked {
                 Some(set) if !self.pk_columns.is_empty() => Arc::new(PkBlockFilterExec::new(
                     scan,
                     self.pk_columns.clone(),
                     set,
-                    n_needed.unwrap_or(0),
+                    0,
                 ))
                     as Arc<dyn ExecutionPlan>,
                 _ => scan,
@@ -338,11 +321,12 @@ impl LsmScanPlanner {
 
                 scanner.create_plan().await
             }
-            LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset = open_flushed_dataset(
+            LsmDataSource::SsTable { path, .. } => {
+                let dataset = open_sstable(
                     path,
                     self.session.as_ref(),
-                    self.flushed_cache.as_ref(),
+                    self.store_params.as_ref(),
+                    self.sstable_cache.as_ref(),
                     self.warmer.as_ref(),
                 )
                 .await?;
@@ -368,7 +352,7 @@ impl LsmScanPlanner {
                 if let Some(expr) = effective {
                     scanner.filter_expr(expr.clone());
                 }
-                // Per-source limit pushdown: flushed generations are
+                // Per-source limit pushdown: SSTables are
                 // within-gen live (dedup-on-flush deletion vectors), so any
                 // `fetch` post-filter rows are valid contributions.
                 if let Some(fetch) = fetch {
@@ -472,10 +456,10 @@ mod tests {
         let shard_id = uuid::Uuid::new_v4();
         let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(5)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
 
-        assert_eq!(snapshot.flushed_generations.len(), 2);
+        assert_eq!(snapshot.sstables.len(), 2);
         assert_eq!(snapshot.current_generation, 5);
     }
 }
@@ -535,7 +519,7 @@ mod integration_tests {
     }
 
     /// Create a dataset at the given URI with the provided batches. Also writes
-    /// the standalone PK sidecar (on `id`) so a flushed-generation source can be
+    /// the standalone PK sidecar (on `id`) so an SSTable source can be
     /// probed by the block-list; harmless for a base table (never probed).
     async fn create_dataset(uri: &str, batches: Vec<RecordBatch>) -> Dataset {
         let schema = batches[0].schema();
@@ -568,8 +552,8 @@ mod integration_tests {
 
     /// Setup a multi-level LSM structure with:
     /// - Base table: ids 1-5 with "base" prefix
-    /// - Flushed gen1: ids 3,4 (updates) with "gen1" prefix
-    /// - Flushed gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix
+    /// - SSTable gen1: ids 3,4 (updates) with "gen1" prefix
+    /// - SSTable gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix
     /// - Active memtable: ids 5,6 (updates) + id 7 (new) with "active" prefix
     ///
     /// Expected deduplication results:
@@ -596,13 +580,13 @@ mod integration_tests {
         let base_batch = create_test_batch(&schema, &[1, 2, 3, 4, 5], "base");
         let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await);
 
-        // Create flushed gen1 as a separate dataset
+        // Create SSTable gen1 as a separate dataset
         let shard_id = Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1");
         create_dataset(&gen1_uri, vec![gen1_batch]).await;
 
-        // Create flushed gen2 as a separate dataset
+        // Create SSTable gen2 as a separate dataset
         let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
         let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2");
         create_dataset(&gen2_uri, vec![gen2_batch]).await;
@@ -610,8 +594,8 @@ mod integration_tests {
         // Build shard snapshot
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
 
         // Create active memtable
         let (batch_store, index_store) =
@@ -835,11 +819,11 @@ mod integration_tests {
     }
 
     /// Regression for the concurrent-read-vs-flush hole: a sealed
-    /// (frozen-awaiting-flush) memtable is not yet recorded as a flushed
-    /// generation, but its rows must still be in the scan's read union and
+    /// (frozen-awaiting-flush) memtable is not yet recorded as an
+    /// SSTable, but its rows must still be in the scan's read union and
     /// dedup correctly by generation across the active/frozen seam.
     ///
-    /// Layout: base(0) ids 1-5, flushed gen1 ids 3,4, flushed gen2 ids
+    /// Layout: base(0) ids 1-5, SSTable gen1 ids 3,4, SSTable gen2 ids
     /// 4,5,6, frozen memtable gen3 ids 6,7, active memtable gen4 ids 7,8.
     #[tokio::test]
     async fn test_lsm_scan_frozen_memtable_in_read_union() {
@@ -868,8 +852,8 @@ mod integration_tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(4)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
 
         // Frozen gen3 (sealed, NOT in the manifest) and active gen4.
         let (frozen_store, frozen_index) =
@@ -928,7 +912,7 @@ mod integration_tests {
         assert_eq!(results.get(&3), Some(&"gen1_3".to_string()));
         assert_eq!(results.get(&4), Some(&"gen2_4".to_string()));
         assert_eq!(results.get(&5), Some(&"gen2_5".to_string()));
-        // id=6: in flushed gen2 AND frozen gen3 -> frozen wins. This is the
+        // id=6: in SSTable gen2 AND frozen gen3 -> frozen wins. This is the
         // bug: pre-fix the frozen memtable fell out of the read union and
         // id=6 resolved to "gen2_6".
         assert_eq!(results.get(&6), Some(&"frozen_6".to_string()));
@@ -996,6 +980,68 @@ mod integration_tests {
         assert_eq!(total_rows, 3, "Should have 3 rows due to limit");
     }
 
+    #[tokio::test]
+    async fn test_lsm_scan_limit_offset_refills_after_update_shadow_across_fragments() {
+        let schema = create_pk_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+        let base_batch = create_test_batch(&schema, &[1, 2, 3, 4], "base");
+        let reader = RecordBatchIterator::new([Ok(base_batch)], schema.clone());
+        let base = Arc::new(
+            Dataset::write(
+                reader,
+                &base_uri,
+                Some(WriteParams {
+                    max_rows_per_file: 2,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+        assert_eq!(
+            base.get_fragments().len(),
+            2,
+            "the stale prefix and live rows must occupy separate fragments"
+        );
+
+        // The newest values for ids 1 and 2 no longer match the predicate, so
+        // their matching base values are shadowed.  The second base fragment
+        // still contains the live matches that must fill offset + limit.
+        let (batch_store, index_store) =
+            pk_indexed(&[create_test_batch(&schema, &[1, 2], "active")]);
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema,
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            )
+            .filter("name LIKE 'base%'")
+            .unwrap()
+            .limit(Some(1), Some(1))
+            .unwrap();
+
+        let batch = scanner.try_into_batch().await.unwrap();
+        let ids = batch
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(
+            ids.iter().collect::<Vec<_>>(),
+            vec![Some(4)],
+            "offset=1, limit=1 must skip id=3 after shadow filtering"
+        );
+    }
+
     #[tokio::test]
     async fn test_lsm_scan_with_offset_without_limit() {
         let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
@@ -1030,7 +1076,7 @@ mod integration_tests {
         let (base_dataset, _, _, pk_columns, _temp_path) = setup_multi_level_lsm().await;
 
         // Create scanner with only base table (no shard snapshots or active memtable)
-        let scanner = LsmScanner::new(base_dataset, vec![], pk_columns);
+        let scanner = LsmScanner::new(base_dataset.clone(), vec![], pk_columns.clone());
 
         let plan = scanner.create_plan().await.unwrap();
 
@@ -1044,6 +1090,22 @@ mod integration_tests {
         .await
         .unwrap();
 
+        // A base-only source has no cross-generation block filter, so its
+        // finite limit remains safe to push into the physical Lance read.
+        let scanner = LsmScanner::new(base_dataset, vec![], pk_columns)
+            .limit(Some(3), None)
+            .unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        assert_plan_node_equals(
+            plan,
+            "GlobalLimitExec: skip=0, fetch=3
+  ProjectionExec:...
+    LocalLimitExec: fetch=3
+      LanceRead:...base/data...range_before=Some(0..3)...refine_filter=--",
+        )
+        .await
+        .unwrap();
+
         // Execute and verify all 5 base rows are returned
         let scanner = LsmScanner::new(
             Arc::new(
@@ -1067,7 +1129,7 @@ mod integration_tests {
     }
 
     #[tokio::test]
-    async fn test_lsm_scan_flushed_only_no_active() {
+    async fn test_lsm_scan_sstable_only_no_active() {
         let (base_dataset, shard_snapshots, _, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
@@ -1229,7 +1291,7 @@ mod integration_tests {
     ///
     /// Similar to setup_multi_level_lsm but:
     /// - Active memtable has a BTree index on the `id` column
-    /// - Flushed datasets have BTree index created (enabling ScalarIndexQuery)
+    /// - SSTables have BTree index created (enabling ScalarIndexQuery)
     async fn setup_multi_level_lsm_with_btree_index() -> (
         Arc<Dataset>,
         Vec<ShardSnapshot>,
@@ -1259,7 +1321,7 @@ mod integration_tests {
         // Reload dataset to pick up the index
         let base_dataset = Arc::new(Dataset::open(&base_uri).await.unwrap());
 
-        // Create flushed gen1 with BTree index
+        // Create SSTable gen1 with BTree index
         let shard_id = Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1");
@@ -1268,7 +1330,7 @@ mod integration_tests {
             .await
             .unwrap();
 
-        // Create flushed gen2 with BTree index
+        // Create SSTable gen2 with BTree index
         let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
         let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2");
         let mut gen2_dataset = create_dataset(&gen2_uri, vec![gen2_batch]).await;
@@ -1279,8 +1341,8 @@ mod integration_tests {
         // Build shard snapshot
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
 
         // Create active memtable with BTree index
         let batch_store = Arc::new(BatchStore::with_capacity(100));
@@ -1615,7 +1677,7 @@ mod integration_tests {
         let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
-        // Use the same base URI the flushed generations were created under, so
+        // Use the same base URI the SSTables were created under, so
         // relative `gen_N` folders resolve to real datasets on disk.
         let base_uri = base_dataset.uri().to_string();
         let arrow_schema: arrow_schema::Schema = base_dataset.schema().into();
@@ -1640,7 +1702,7 @@ mod integration_tests {
         );
         assert!(
             plan_str.contains("gen_1") && plan_str.contains("gen_2"),
-            "Plan must scan flushed generations, got: {}",
+            "Plan must scan SSTables, got: {}",
             plan_str
         );
         assert!(
@@ -1747,8 +1809,8 @@ mod integration_tests {
     }
 
     #[tokio::test]
-    async fn test_lsm_scan_without_base_table_no_flushed_no_active() {
-        // No base, no flushed, no active → empty result, valid plan.
+    async fn test_lsm_scan_without_base_table_no_sstable_no_active() {
+        // No base, no SSTable, no active → empty result, valid plan.
         let schema = create_pk_schema();
         let scanner = LsmScanner::without_base_table(
             schema,
@@ -2082,8 +2144,48 @@ mod integration_tests {
     }
 
     #[tokio::test]
-    async fn test_lsm_scan_flushed_tombstone_masks_base() {
-        // A tombstone living in a flushed generation masks the older base row by
+    async fn test_lsm_scan_limit_refills_after_active_tombstone_shadows_base() {
+        let base_schema = create_pk_schema();
+        let mem_schema = ts_pk_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+        let base = Arc::new(
+            create_dataset(
+                &base_uri,
+                vec![create_test_batch(&base_schema, &[1, 2, 3], "base")],
+            )
+            .await,
+        );
+
+        let active_batch = ts_batch(&mem_schema, &[(1, None, true)]);
+        let (batch_store, index_store) = pk_indexed(&[active_batch]);
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema: mem_schema,
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            )
+            .limit(Some(2), None)
+            .unwrap();
+
+        let batch = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            collect_sorted_ids(&[batch]),
+            vec![2, 3],
+            "the base scan must continue past the shadowed row to fill the limit"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_lsm_scan_sstable_tombstone_masks_base() {
+        // A tombstone living in an SSTable masks the older base row by
         // PK presence (block-list) and is itself dropped by the folded predicate.
         let base_schema = create_pk_schema();
         let mem_schema = ts_pk_schema();
@@ -2098,14 +2200,14 @@ mod integration_tests {
             .await,
         );
 
-        // Flushed gen 1 holds only a tombstone for id=2 (written with the
-        // `_tombstone` schema, so the flushed arm folds `NOT _tombstone`).
+        // SSTable gen 1 holds only a tombstone for id=2 (written with the
+        // `_tombstone` schema, so the SSTable arm folds `NOT _tombstone`).
         let shard_id = Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         create_dataset(&gen1_uri, vec![ts_batch(&mem_schema, &[(2, None, true)])]).await;
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         let scanner = LsmScanner::new(base, vec![shard_snapshot], vec!["id".to_string()]);
         let batches: Vec<RecordBatch> = scanner
@@ -2118,13 +2220,13 @@ mod integration_tests {
         assert_eq!(
             collect_sorted_ids(&batches),
             vec![1, 3],
-            "id=2 deleted via flushed-generation tombstone"
+            "id=2 deleted via an SSTable tombstone"
         );
     }
 
     #[tokio::test]
     async fn test_lsm_scan_tombstone_does_not_consume_limit() {
-        // A single (newest) flushed generation holds both tombstones and live
+        // A single (newest) SSTable holds both tombstones and live
         // rows. With LIMIT 2 the folded `NOT _tombstone` runs *before* the
         // per-source pushdown limit, so the limit counts only live rows — we get
         // 2 live rows, not 0 (which is what a post-limit tombstone filter, or a
@@ -2152,7 +2254,7 @@ mod integration_tests {
         .await;
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         let scanner = LsmScanner::without_base_table(
             base_schema,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index 4f7c5f093f6..9fc44290209 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -34,12 +34,13 @@ use crate::dataset::mem_wal::memtable::batch_store::BatchStore;
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
 use super::exec::{BloomFilterGuardExec, CoalesceFirstExec, compute_pk_hash_from_scalars};
-use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
     DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns,
     project_to_canonical, validate_projection_names, wants_row_address, wants_row_id,
 };
+use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable};
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Plans point lookup queries over LSM data.
 ///
@@ -87,12 +88,14 @@ pub struct LsmPointLookupPlanner {
     /// Bloom filters for each memtable generation.
     /// Map: generation -> bloom filter
     bloom_filters: std::collections::HashMap<u64, Arc<Sbbf>>,
-    /// Session threaded into flushed-generation opens (shared caches).
+    /// Session threaded into SSTable opens (shared caches).
     session: Option<Arc<Session>>,
-    /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<dyn DatasetCache>>,
-    /// Optional warmer fired on first open of a flushed generation.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Store params for opening SSTables, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
+    /// Cache of opened SSTable datasets.
+    sstable_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of an SSTable.
+    warmer: Option<Arc<dyn SsTableWarmer>>,
     /// Precomputed canonical output schema for the no-projection case, so the
     /// hot `lookup(.., None)` path clones an `Arc` instead of rebuilding the
     /// schema on every call.
@@ -124,32 +127,38 @@ impl LsmPointLookupPlanner {
             base_schema,
             bloom_filters: std::collections::HashMap::new(),
             session: None,
-            flushed_cache: None,
+            store_params: None,
+            sstable_cache: None,
             warmer: None,
             none_target,
             task_ctx: SessionContext::new().task_ctx(),
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open SSTables.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
-    /// Inject a cache of opened flushed-generation datasets, making repeated
+    /// Set the store params used to open SSTables.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
+    /// Inject a cache of opened SSTable datasets, making repeated
     /// lookups against the same generation a pure `Arc::clone`. Populate it up
     /// front during scan setup via
     /// [`DatasetMemWalExt::prewarm_mem_wal`](crate::dataset::mem_wal::DatasetMemWalExt::prewarm_mem_wal)
     /// so the first gen-key lookup does not pay the dataset open.
-    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
-        self.flushed_cache = Some(cache);
+    pub fn with_sstable_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
+        self.sstable_cache = Some(cache);
         self
     }
 
-    /// Inject the warmer fired on first open of a flushed generation.
-    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+    /// Inject the warmer fired on first open of an SSTable.
+    pub fn with_warmer(mut self, warmer: Arc<dyn SsTableWarmer>) -> Self {
         self.warmer = Some(warmer);
         self
     }
@@ -332,7 +341,7 @@ impl LsmPointLookupPlanner {
     /// For a single-column primary key this probes the in-memory memtables'
     /// BTree index directly — no DataFusion plan — newest generation first, and
     /// returns on the first hit. Only when the lookup must consult an on-disk
-    /// source (a flushed generation or the base table), a memtable lacks a
+    /// source (an SSTable or the base table), a memtable lacks a
     /// BTree on the key, the key is multi-column, or the projection requests
     /// system columns does it fall back to [`Self::plan_lookup`]. The result is
     /// identical to executing `plan_lookup` and taking the first row; the fast
@@ -407,7 +416,7 @@ impl LsmPointLookupPlanner {
                     None => {
                         // Every in-memory memtable missed. If there is no
                         // on-disk source, the key does not exist; otherwise the
-                        // plan path consults the base table / flushed gens.
+                        // plan path consults the base table / SSTables.
                         if !self.collector.has_on_disk_sources() {
                             return Ok(None);
                         }
@@ -645,13 +654,18 @@ impl LsmPointLookupPlanner {
                     scanner.with_row_address();
                 }
                 scanner.filter_expr(filter.clone());
-                scanner.create_plan().await?
+                // Box at the call site: `create_plan`'s inlined async layout exceeds
+                // rustc's depth limit up this point-lookup chain, and boxing inside
+                // `create_plan` instead triggers a `Box<Future>: Send` solver overflow
+                // (E0275 downstream). Same for the other arms below.
+                Box::pin(scanner.create_plan()).await?
             }
-            LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset = open_flushed_dataset(
+            LsmDataSource::SsTable { path, .. } => {
+                let dataset = open_sstable(
                     path,
                     self.session.as_ref(),
-                    self.flushed_cache.as_ref(),
+                    self.store_params.as_ref(),
+                    self.sstable_cache.as_ref(),
                     self.warmer.as_ref(),
                 )
                 .await?;
@@ -662,7 +676,7 @@ impl LsmPointLookupPlanner {
                 let cols = cols_with_tombstone(&cols, dataset.schema().field(TOMBSTONE).is_some());
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
                 scanner.filter_expr(filter.clone());
-                scanner.create_plan().await?
+                Box::pin(scanner.create_plan()).await?
             }
             LsmDataSource::ActiveMemTable {
                 batch_store,
@@ -685,7 +699,7 @@ impl LsmPointLookupPlanner {
                 // over insert-ordered scan would return the *oldest* of
                 // multiple rows sharing the target primary key.
                 scanner.with_row_id();
-                let raw = scanner.create_plan().await?;
+                let raw = Box::pin(scanner.create_plan()).await?;
                 // The filter already restricts to the exact PK value, so the
                 // scan yields that key's insert history. Within the active
                 // memtable larger `_rowid` = newer insert, so sorting `_rowid`
@@ -881,7 +895,12 @@ fn probe_position(
     if len == 0 {
         return Ok(ProbePos::Miss);
     }
-    let last_visible_idx = index_store.max_visible_batch_position().min(len - 1);
+    // The cursor is an exclusive count, so the last visible batch sits at
+    // `count - 1`. A count of 0 means nothing is visible yet — not "batch 0".
+    let visible_count = index_store.visible_count().min(len);
+    let Some(last_visible_idx) = visible_count.checked_sub(1) else {
+        return Ok(ProbePos::Miss);
+    };
     let last = batch_store.get(last_visible_idx).ok_or_else(|| {
         lance_core::Error::internal("point-lookup: visible batch index out of range")
     })?;
@@ -1100,7 +1119,7 @@ mod tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         // Create collector
         let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]);
@@ -1181,10 +1200,10 @@ mod tests {
         let base_path = temp_dir.path().to_str().unwrap();
 
         // No base dataset is created. We still need a base URI so the collector
-        // can resolve flushed-generation paths.
+        // can resolve SSTable paths.
         let base_uri = format!("{}/base", base_path);
 
-        // Create a flushed generation under {base_uri}/_mem_wal/{shard}/gen_1
+        // Create an SSTable under {base_uri}/_mem_wal/{shard}/gen_1
         let shard_id = Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let gen1_batch = create_test_batch(&schema, &[2, 3], "gen1");
@@ -1192,12 +1211,12 @@ mod tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]);
         let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema);
 
-        // id=3 lives in the flushed generation
+        // id=3 lives in the SSTable
         let pk_values = vec![ScalarValue::Int32(Some(3))];
         let plan = planner.plan_lookup(&pk_values, None).await.unwrap();
 
@@ -1378,8 +1397,9 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
-        // BTree on the PK so that `max_visible_batch_position` advances as
-        // we insert, otherwise the scanner sees no batches at all.
+        // BTree on the PK: the point lookup resolves keys through the indexed PK
+        // path, which this exercises. (`indexed_count`/`visible_count` advance
+        // from the batch position regardless of whether any index is configured.)
         index_store.add_btree("id_idx".to_string(), 0, "id".to_string());
 
         // Two writes to pk=1, then an unrelated pk=2. The "new" row goes
@@ -1520,10 +1540,10 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_point_lookup_flushed_memtable_returns_newest_duplicate() {
-        // Regression / invariant pin: when a flushed memtable contains two
+    async fn test_point_lookup_sstable_returns_newest_duplicate() {
+        // Regression / invariant pin: when an SSTable contains two
         // rows for the same PK, the lookup must return the newer one. The
-        // flushed dataset is reverse-written (newest at the smallest
+        // SSTable dataset is reverse-written (newest at the smallest
         // physical position), so we simulate that here by writing the
         // dataset with the new row first. The point-lookup plan today
         // returns the first match (smallest `_rowid`) under reverse-write,
@@ -1544,7 +1564,7 @@ mod tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]);
         let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema);
@@ -1564,7 +1584,7 @@ mod tests {
         assert_eq!(
             name_arr.value(0),
             "new_1",
-            "flushed-arm lookup must return the row at the smallest _rowid (newest under reverse-write)"
+            "SSTable-arm lookup must return the row at the smallest _rowid (newest under reverse-write)"
         );
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/scanner/projection.rs b/rust/lance/src/dataset/mem_wal/scanner/projection.rs
index 0ec482aebf8..a39c83a8f1b 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/projection.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/projection.rs
@@ -6,8 +6,8 @@
 //!
 //! `MemTableScanner::project()` only special-cases `_rowid`; passing other
 //! system columns through it errors. And cross-LSM values for system
-//! columns aren't comparable (a `_rowid` of 5 in the base and in a flushed
-//! memtable refer to different rows).
+//! columns aren't comparable (a `_rowid` of 5 in the base and in an
+//! SSTable refer to different rows).
 //!
 //! - [`build_scanner_projection`] — strips system / `_distance` cols, appends PKs.
 //! - [`canonical_output_schema`] — final schema honoring user order; system
diff --git a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs
similarity index 75%
rename from rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs
rename to rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs
index 7a5280bedb8..9bb8dadcf02 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-//! Cache of opened flushed-generation datasets for the LSM scanner.
+//! Cache of opened SSTable datasets for the LSM scanner.
 //!
-//! Flushed generations are written exactly once to a globally-unique,
+//! SSTables are written exactly once to a globally-unique,
 //! content-addressed path (see `memtable/flush.rs`): a fresh random hash per
 //! flush invocation means the same path always maps to the same immutable
 //! bytes. A cached `Arc<Dataset>` therefore can never go stale and needs no
@@ -11,11 +11,11 @@
 //! optimization driven by the consumer at compaction time.
 //!
 //! ```text
-//! query ──> open_flushed_dataset(path, session, cache)
+//! query ──> open_sstable(path, session, cache)
 //!                                  │
 //!            cache.is_some() ──────┤────── cache.is_none()
 //!                  │                              │
-//!     FlushedMemTableCache::get_or_open      DatasetBuilder::from_uri
+//!     SsTableCache::get_or_open      DatasetBuilder::from_uri
 //!     (single-flight, shared Arc)           (cold open every call)
 //! ```
 
@@ -24,31 +24,41 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use lance_core::{Error, Result};
+use lance_io::object_store::ObjectStoreParams;
 
 use crate::dataset::{Dataset, DatasetBuilder};
 use crate::session::Session;
 
-/// Cache of opened flushed-generation datasets, keyed by resolved path.
+/// Cache of opened SSTable datasets, keyed by resolved path.
 ///
-/// Flushed generations live at a globally-unique, immutable path, so cached
+/// SSTables live at a globally-unique, immutable path, so cached
 /// entries are never stale and require no TTL. Intended to be held by a
 /// long-lived owner (one per process or per table) and injected into
 /// per-request scanners via [`crate::dataset::mem_wal::scanner::LsmScanner`]
 /// (and the point-lookup / vector-search planners).
 ///
-/// The key is the resolved absolute flushed path
+/// The key is the resolved absolute SSTable path
 /// (`{base}/_mem_wal/{shard}/{folder}`), which is globally unique, so a single
 /// cache can safely span multiple tables.
-pub struct FlushedMemTableCache {
+///
+/// `store_params` is deliberately *not* part of the key: the first caller to
+/// open a path binds the store that every later hit reuses. Credential rotation
+/// still works — a vended-credential store holds the live
+/// `StorageOptionsAccessor` and re-resolves per request, so a cached handle
+/// never carries expired credentials. What this does assume is that a given
+/// path is only ever served under one store configuration. Serving one table
+/// through a single cache under two different `ObjectStoreParams` would hand
+/// every caller the store the first one opened with.
+pub struct SsTableCache {
     // `moka`'s async cache gives a bounded size plus single-flight
     // `try_get_with`, so concurrent first-queries on a just-flushed
-    // generation open the dataset exactly once. The opened dataset carries the
+    // SSTable open the dataset exactly once. The opened dataset carries the
     // session index cache, which also backs each generation's standalone PK
     // dedup index (see `block_list::open_pk_index`) — no separate cache path.
     inner: moka::future::Cache<String, Arc<Dataset>>,
 }
 
-impl FlushedMemTableCache {
+impl SsTableCache {
     /// Create a cache holding at most `max_entries` opened datasets.
     ///
     /// Eviction is size-only (no TTL): an evicted-then-re-requested generation
@@ -66,16 +76,14 @@ impl FlushedMemTableCache {
     }
 
     /// Get the dataset for `path`, opening it (exactly once) on a miss.
-    ///
-    /// `session` is threaded into the open so the first open populates the
-    /// shared index / file-metadata caches; subsequent hits are a pure
-    /// `Arc::clone` with zero object-store I/O. Concurrent callers for the
-    /// same path share a single open via `moka`'s single-flight
-    /// `try_get_with`.
+    /// Concurrent callers share a single open via `moka`'s single-flight
+    /// `try_get_with`; hits are a pure `Arc::clone`. `session` / `store_params`
+    /// configure the open.
     pub async fn get_or_open(
         &self,
         path: &str,
         session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
     ) -> Result<Arc<Dataset>> {
         self.inner
             .try_get_with(path.to_string(), async move {
@@ -83,6 +91,9 @@ impl FlushedMemTableCache {
                 if let Some(session) = session {
                     builder = builder.with_session(session);
                 }
+                if let Some(store_params) = store_params {
+                    builder = builder.with_store_params(store_params);
+                }
                 builder.load().await.map(Arc::new)
             })
             .await
@@ -108,36 +119,46 @@ impl FlushedMemTableCache {
     }
 }
 
-impl std::fmt::Debug for FlushedMemTableCache {
+impl std::fmt::Debug for SsTableCache {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("FlushedMemTableCache")
+        f.debug_struct("SsTableCache")
             .field("entry_count", &self.inner.entry_count())
             .finish()
     }
 }
 
-/// Caching of opened flushed-generation datasets, keyed by immutable path. The
+/// Caching of opened SSTable datasets, keyed by immutable path. The
 /// opened dataset carries the session index cache, which also backs each
 /// generation's secondary indexes and its PK dedup sidecar (see
 /// `block_list::open_pk_index`) — so a single `get_or_open` is the
-/// whole caching surface. Implemented by [`FlushedMemTableCache`]; a
-/// [`GenerationWarmer`] composes one to warm through it, and a consumer may
+/// whole caching surface. Implemented by [`SsTableCache`]; a
+/// [`SsTableWarmer`] composes one to warm through it, and a consumer may
 /// supply its own implementation.
 #[async_trait]
 pub trait DatasetCache: Send + Sync + std::fmt::Debug {
-    async fn get_or_open(&self, path: &str, session: Option<Arc<Session>>) -> Result<Arc<Dataset>>;
+    async fn get_or_open(
+        &self,
+        path: &str,
+        session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Arc<Dataset>>;
 
     /// Drop cached entries whose path is not in `live_paths`. Async so an
     /// implementation can evict retired generations' index objects (e.g.
     /// `Session::invalidate_index_prefix`) without a later breaking signature
-    /// change; [`FlushedMemTableCache`]'s own eviction is synchronous.
+    /// change; [`SsTableCache`]'s own eviction is synchronous.
     async fn retain_paths(&self, live_paths: &HashSet<String>);
 }
 
 #[async_trait]
-impl DatasetCache for FlushedMemTableCache {
-    async fn get_or_open(&self, path: &str, session: Option<Arc<Session>>) -> Result<Arc<Dataset>> {
-        Self::get_or_open(self, path, session).await
+impl DatasetCache for SsTableCache {
+    async fn get_or_open(
+        &self,
+        path: &str,
+        session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Arc<Dataset>> {
+        Self::get_or_open(self, path, session, store_params).await
     }
 
     async fn retain_paths(&self, live_paths: &HashSet<String>) {
@@ -145,7 +166,7 @@ impl DatasetCache for FlushedMemTableCache {
     }
 }
 
-/// Proactively warms a flushed generation into the shared caches: open the
+/// Proactively warms an SSTable into the shared caches: open the
 /// dataset and pre-load its secondary indexes and PK dedup sidecar so the first
 /// query sees no cold reads. This is the **seam** the flush and read paths fire
 /// — lance defines it; the consumer (e.g. the WAL pod) implements it. `None` =>
@@ -161,34 +182,42 @@ impl DatasetCache for FlushedMemTableCache {
 /// and cheap when the path is already warm** (e.g. dedup in-flight and
 /// completed paths) — a redundant call must not re-do work or fail.
 #[async_trait]
-pub trait GenerationWarmer: Send + Sync + std::fmt::Debug {
+pub trait SsTableWarmer: Send + Sync + std::fmt::Debug {
     async fn warm(&self, path: &str) -> Result<()>;
 }
 
-/// Open a flushed-generation dataset, shared by all three LSM open sites
+/// Open an SSTable dataset, shared by all three LSM open sites
 /// (scan, point lookup, vector search).
 ///
 /// - `cache` present: route through a [`DatasetCache`] (e.g.
-///   [`FlushedMemTableCache`]: single-flight, shared `Arc`, manifest read
+///   [`SsTableCache`]: single-flight, shared `Arc`, manifest read
 ///   amortized across queries).
 /// - `cache` absent: cold open via [`DatasetBuilder`]. Passing `session`
 ///   still reuses the shared index / metadata caches; `None`/`None`
 ///   reproduces the original per-query cold-open behavior exactly.
 /// - `warmer` present: fire a fire-and-forget warm-on-open backstop behind the
 ///   returned handle (the warmer dedups already-warm paths). `None` => no warming.
-pub async fn open_flushed_dataset(
+pub async fn open_sstable(
     path: &str,
     session: Option<&Arc<Session>>,
+    store_params: Option<&ObjectStoreParams>,
     cache: Option<&Arc<dyn DatasetCache>>,
-    warmer: Option<&Arc<dyn GenerationWarmer>>,
+    warmer: Option<&Arc<dyn SsTableWarmer>>,
 ) -> Result<Arc<Dataset>> {
     let dataset = match cache {
-        Some(cache) => cache.get_or_open(path, session.cloned()).await?,
+        Some(cache) => {
+            cache
+                .get_or_open(path, session.cloned(), store_params.cloned())
+                .await?
+        }
         None => {
             let mut builder = DatasetBuilder::from_uri(path);
             if let Some(session) = session {
                 builder = builder.with_session(session.clone());
             }
+            if let Some(store_params) = store_params {
+                builder = builder.with_store_params(store_params.clone());
+            }
             Arc::new(builder.load().await?)
         }
     };
@@ -238,9 +267,9 @@ mod tests {
         let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap());
         write_dataset(&uri, &[1, 2, 3]).await;
 
-        let cache = FlushedMemTableCache::new(8);
-        let first = cache.get_or_open(&uri, None).await.unwrap();
-        let second = cache.get_or_open(&uri, None).await.unwrap();
+        let cache = SsTableCache::new(8);
+        let first = cache.get_or_open(&uri, None, None).await.unwrap();
+        let second = cache.get_or_open(&uri, None, None).await.unwrap();
 
         assert!(
             Arc::ptr_eq(&first, &second),
@@ -261,7 +290,7 @@ mod tests {
         let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap());
         write_dataset(&uri, &[1, 2, 3]).await;
 
-        let cache = Arc::new(FlushedMemTableCache::new(8));
+        let cache = Arc::new(SsTableCache::new(8));
         let calls = Arc::new(AtomicUsize::new(0));
 
         let mut handles = Vec::new();
@@ -271,7 +300,7 @@ mod tests {
             let calls = calls.clone();
             handles.push(tokio::spawn(async move {
                 calls.fetch_add(1, Ordering::SeqCst);
-                cache.get_or_open(&uri, None).await.unwrap()
+                cache.get_or_open(&uri, None, None).await.unwrap()
             }));
         }
 
@@ -298,9 +327,9 @@ mod tests {
         write_dataset(&keep_uri, &[1]).await;
         write_dataset(&drop_uri, &[2]).await;
 
-        let cache = FlushedMemTableCache::new(8);
-        cache.get_or_open(&keep_uri, None).await.unwrap();
-        cache.get_or_open(&drop_uri, None).await.unwrap();
+        let cache = SsTableCache::new(8);
+        cache.get_or_open(&keep_uri, None, None).await.unwrap();
+        cache.get_or_open(&drop_uri, None, None).await.unwrap();
         cache.inner.run_pending_tasks().await;
         assert_eq!(cache.inner.entry_count(), 2);
 
@@ -314,15 +343,15 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_open_flushed_dataset_no_cache_matches_direct_open() {
+    async fn test_open_sstable_no_cache_matches_direct_open() {
         // The `None`/`None` path must reproduce a plain cold open: same data,
         // independent Arc per call (no caching).
         let temp_dir = tempfile::tempdir().unwrap();
         let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap());
         write_dataset(&uri, &[7, 8, 9]).await;
 
-        let a = open_flushed_dataset(&uri, None, None, None).await.unwrap();
-        let b = open_flushed_dataset(&uri, None, None, None).await.unwrap();
+        let a = open_sstable(&uri, None, None, None, None).await.unwrap();
+        let b = open_sstable(&uri, None, None, None, None).await.unwrap();
         assert!(
             !Arc::ptr_eq(&a, &b),
             "no-cache path must cold-open each call"
@@ -330,11 +359,11 @@ mod tests {
         assert_eq!(a.count_rows(None).await.unwrap(), 3);
 
         // With a cache, the second call is a shared clone.
-        let cache: Arc<dyn DatasetCache> = Arc::new(FlushedMemTableCache::new(8));
-        let c = open_flushed_dataset(&uri, None, Some(&cache), None)
+        let cache: Arc<dyn DatasetCache> = Arc::new(SsTableCache::new(8));
+        let c = open_sstable(&uri, None, None, Some(&cache), None)
             .await
             .unwrap();
-        let d = open_flushed_dataset(&uri, None, Some(&cache), None)
+        let d = open_sstable(&uri, None, None, Some(&cache), None)
             .await
             .unwrap();
         assert!(Arc::ptr_eq(&c, &d), "cached path must reuse the Arc");
@@ -348,7 +377,7 @@ mod tests {
     }
 
     #[async_trait]
-    impl GenerationWarmer for NotifyingWarmer {
+    impl SsTableWarmer for NotifyingWarmer {
         async fn warm(&self, _path: &str) -> Result<()> {
             self.calls.fetch_add(1, Ordering::SeqCst);
             self.notify.notify_one();
@@ -357,7 +386,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_open_flushed_dataset_fires_warm_on_open() {
+    async fn test_open_sstable_fires_warm_on_open() {
         // The warm-on-open backstop fires the warmer (fire-and-forget) when a
         // generation is opened, so generations the flusher never warmed still
         // get warmed lazily on first read.
@@ -367,12 +396,12 @@ mod tests {
 
         let calls = Arc::new(AtomicUsize::new(0));
         let notify = Arc::new(tokio::sync::Notify::new());
-        let warmer: Arc<dyn GenerationWarmer> = Arc::new(NotifyingWarmer {
+        let warmer: Arc<dyn SsTableWarmer> = Arc::new(NotifyingWarmer {
             calls: calls.clone(),
             notify: notify.clone(),
         });
 
-        let ds = open_flushed_dataset(&uri, None, None, Some(&warmer))
+        let ds = open_sstable(&uri, None, None, None, Some(&warmer))
             .await
             .unwrap();
         assert_eq!(ds.count_rows(None).await.unwrap(), 3);
diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
index 59f721aa08c..3c314b36a75 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
@@ -28,19 +28,20 @@ use crate::io::exec::TakeExec;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
-use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
     DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns,
     project_to_canonical, validate_projection_names, wants_row_id,
 };
+use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable};
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Plans vector search queries over LSM data.
 ///
 /// Each source is independently newest-per-PK before the union — the active
 /// memtable via exact brute-force KNN when PK rewrites or a filter require it
 /// (append-only active data can still use HNSW),
-/// flushed generations via their within-generation deletion vector — and the
+/// SSTables via their within-generation deletion vector — and the
 /// cross-generation block-list ([`super::exec::PkBlockFilterExec`]) drops any
 /// PK superseded by a newer generation. So each PK reaches the union from
 /// exactly one source and a distance-ordered merge yields the global top-k; no
@@ -58,9 +59,9 @@ use crate::session::Session;
 ///               MemTableBruteForceVectorExec or VectorIndexExec: active memtable KNN
 ///         ProjectionExec (canonical output schema)
 ///           ProjectionExec (null_columns _rowid)
-///             PkBlockFilterExec: block-list                   (flushed)
-///               KNNExec: flushed gen N, fetch=ceil(k*overfetch) (fast_search)
-///         … one per flushed gen …
+///             PkBlockFilterExec: block-list                   (SSTable)
+///               KNNExec: SSTable gen N, fetch=ceil(k*overfetch) (fast_search)
+///         … one per SSTable gen …
 ///         ProjectionExec (canonical output schema)
 ///           PkBlockFilterExec: block-list                     (base)
 ///             KNNExec: base table, k (fast_search)[.refine()?]
@@ -68,11 +69,11 @@ use crate::session::Session;
 ///
 /// # Index-Only Search (fast_search)
 ///
-/// For base table and flushed memtables we use `fast_search()` to only
+/// For base table and SSTables we use `fast_search()` to only
 /// search indexed data. This is correct because:
-/// - Each flushed memtable has its own vector index built during flush.
+/// - Each SSTable has its own vector index built during flush.
 /// - The active memtable covers any unindexed data.
-/// - Searching unindexed data in base/flushed would be redundant.
+/// - Searching unindexed data in base/SSTable would be redundant.
 pub struct LsmVectorSearchPlanner {
     /// Data source collector.
     collector: LsmDataSourceCollector,
@@ -89,15 +90,17 @@ pub struct LsmVectorSearchPlanner {
     /// the per-source KNN output. Memtable rows already carry all columns;
     /// the take only fetches additional data for base rows (real `_rowid`).
     dataset: Option<Arc<Dataset>>,
-    /// Session threaded into flushed-generation opens (shared caches).
+    /// Session threaded into SSTable opens (shared caches).
     session: Option<Arc<Session>>,
-    /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<dyn DatasetCache>>,
-    /// Optional warmer fired on first open of a flushed generation.
-    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Store params for opening SSTables, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
+    /// Cache of opened SSTable datasets.
+    sstable_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of an SSTable.
+    warmer: Option<Arc<dyn SsTableWarmer>>,
     /// Optional prefilter predicate applied to every source arm before its KNN
     /// search, so rows failing the predicate never enter the top-k. Base and
-    /// flushed arms use the dataset scanner's native prefilter; memtable arms
+    /// SSTable arms use the dataset scanner's native prefilter; memtable arms
     /// route to a filtered brute-force scan.
     filter: Option<Expr>,
 }
@@ -127,7 +130,8 @@ impl LsmVectorSearchPlanner {
             distance_type,
             dataset: None,
             session: None,
-            flushed_cache: None,
+            store_params: None,
+            sstable_cache: None,
             warmer: None,
             filter: None,
         }
@@ -135,28 +139,33 @@ impl LsmVectorSearchPlanner {
 
     /// Attach an optional prefilter predicate. Every source arm restricts its
     /// KNN to rows matching the predicate (true prefilter), so results match a
-    /// normal filtered vector scan over base ∪ flushed ∪ in-memory data.
+    /// normal filtered vector scan over base ∪ SSTables ∪ in-memory data.
     pub fn with_filter(mut self, filter: Option<Expr>) -> Self {
         self.filter = filter;
         self
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open SSTables.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
-    /// Inject a cache of opened flushed-generation datasets, making repeated
+    /// Set the store params used to open SSTables.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
+    /// Inject a cache of opened SSTable datasets, making repeated
     /// searches against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
-        self.flushed_cache = Some(cache);
+    pub fn with_sstable_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
+        self.sstable_cache = Some(cache);
         self
     }
 
-    /// Inject the warmer fired on first open of a flushed generation.
-    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+    /// Inject the warmer fired on first open of an SSTable.
+    pub fn with_warmer(mut self, warmer: Arc<dyn SsTableWarmer>) -> Self {
         self.warmer = Some(warmer);
         self
     }
@@ -235,7 +244,8 @@ impl LsmVectorSearchPlanner {
         let block_lists = Box::pin(super::block_list::compute_source_block_lists(
             &sources,
             self.session.as_ref(),
-            self.flushed_cache.as_ref(),
+            self.store_params.as_ref(),
+            self.sstable_cache.as_ref(),
         ))
         .await?;
 
@@ -296,8 +306,8 @@ impl LsmVectorSearchPlanner {
             //  * active: append-only memtables can use HNSW directly; once a
             //    PK rewrite is observed, `MemTableBruteForceVectorExec` drops
             //    superseded versions before the top-k cut.
-            //  * flushed/base: drop cross-gen superseded rows via the
-            //    block-list (within-gen is handled by the flushed DV).
+            //  * SSTable/base: drop cross-gen superseded rows via the
+            //    block-list (within-gen is handled by the SSTable DV).
             let knn = match blocked {
                 Some(_) if self.pk_columns.is_empty() => knn,
                 Some(set) => Arc::new(super::exec::PkBlockFilterExec::new(
@@ -442,11 +452,12 @@ impl LsmVectorSearchPlanner {
                 }
                 scanner.create_plan().await
             }
-            LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset = open_flushed_dataset(
+            LsmDataSource::SsTable { path, .. } => {
+                let dataset = open_sstable(
                     path,
                     self.session.as_ref(),
-                    self.flushed_cache.as_ref(),
+                    self.store_params.as_ref(),
+                    self.sstable_cache.as_ref(),
                     self.warmer.as_ref(),
                 )
                 .await?;
@@ -669,7 +680,7 @@ mod tests {
         let dataset = Dataset::write(reader, uri, Some(WriteParams::default()))
             .await
             .unwrap();
-        // Also write the standalone PK sidecar (on `id`) so a flushed-generation
+        // Also write the standalone PK sidecar (on `id`) so an SSTable
         // source can be probed by the block-list (harmless for a base table).
         if has_id {
             crate::dataset::mem_wal::scanner::block_list::write_pk_sidecar(uri, &batches, &["id"])
@@ -847,7 +858,7 @@ mod tests {
             out_cols
         );
         // Internal columns must not leak: `_rowid` (added by Lance's fast_search
-        // in the base/flushed arms) and `_memtable_gen` (added by the LSM merge
+        // in the base/SSTable arms) and `_memtable_gen` (added by the LSM merge
         // when bloom filters are present) are bookkeeping, not API.
         assert!(
             out_schema.field_with_name("_rowid").is_err(),
@@ -1215,13 +1226,13 @@ mod tests {
         );
     }
 
-    /// The flushed arm must also apply the filter as a true prefilter, and that
+    /// The SSTable arm must also apply the filter as a true prefilter, and that
     /// prefiltered candidate set must compose with cross-generation block-list
     /// filtering plus over-fetch. Gen 1's closest predicate-matching row (id=3)
     /// is superseded by gen 2; with over-fetch, gen 1 should still contribute
     /// the next live predicate match (id=4).
     #[tokio::test]
-    async fn test_vector_search_flushed_prefilter_composes_with_block_list() {
+    async fn test_vector_search_sstable_prefilter_composes_with_block_list() {
         use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
         use crate::index::DatasetIndexExt;
         use crate::index::vector::VectorIndexParams;
@@ -1260,8 +1271,8 @@ mod tests {
 
         let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]);
 
         let planner = LsmVectorSearchPlanner::new(
@@ -1286,7 +1297,7 @@ mod tests {
         assert_eq!(rows.len(), 1, "expected one result, got {:?}", rows);
         assert_eq!(
             rows[0].0, 4,
-            "flushed prefilter should return live id=4 after stale id=3 is blocked; got {:?}",
+            "SSTable prefilter should return live id=4 after stale id=3 is blocked; got {:?}",
             rows
         );
     }
@@ -1996,14 +2007,14 @@ mod tests {
     #[tokio::test]
     async fn test_vector_search_dedup_across_generations() {
         // Regression: same primary key inserted into two sources (older
-        // flushed gen and newer active memtable) with different vectors.
-        // Without the cross-source PK dedup the older flushed row would
+        // SSTable gen and newer active memtable) with different vectors.
+        // Without the cross-source PK dedup the older SSTable row would
         // still appear in top-k. The newer-generation row must win.
         //
-        // We simulate a "flushed gen 1" by writing a tiny Lance dataset
+        // We simulate a "SSTable gen 1" by writing a tiny Lance dataset
         // under {base_uri}/_mem_wal/{shard}/gen_1 and pointing the
         // collector at it. Real flush would reverse-write, but for this
-        // test we only have one row in the flushed gen so order is moot.
+        // test we only have one row in the SSTable gen so order is moot.
         use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
         use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
         use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
@@ -2015,7 +2026,7 @@ mod tests {
         let base_path = temp_dir.path().to_str().unwrap();
         let base_uri = format!("{}/base", base_path);
 
-        // Flushed gen 1 holds an older version of pk=1 with a "wrong" vector.
+        // SSTable gen 1 holds an older version of pk=1 with a "wrong" vector.
         let shard_id = uuid::Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
         let old_pk1 = create_test_batch_with_vector(&schema, 1, [9.0, 9.0, 9.0, 9.0]);
@@ -2048,7 +2059,7 @@ mod tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot])
             .with_in_memory_memtables(
                 shard_id,
@@ -2104,7 +2115,7 @@ mod tests {
     async fn test_vector_search_system_columns_real_only_for_base() {
         // Covers three properties of the per-source system columns:
         //   1. base-hit `_rowid`/`_rowaddr` carry real values
-        //   2. flushed-memtable arm runs without erroring
+        //   2. SSTable arm runs without erroring
         //   3. `_rowaddr` symmetry with `_rowid` (same code path, both are
         //      surfaced when requested and NULL'd outside the base arm)
         use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
@@ -2131,7 +2142,7 @@ mod tests {
             .unwrap();
         let base_dataset = Arc::new(base_dataset);
 
-        // Flushed memtable: id=2 (a separate Lance dataset under
+        // SSTable: id=2 (a separate Lance dataset under
         // {base_uri}/_mem_wal/{shard}/gen_1) with its own vector index.
         let shard_id = uuid::Uuid::new_v4();
         let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
@@ -2163,7 +2174,7 @@ mod tests {
 
         let shard_snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(2)
-            .with_flushed_generation(1, "gen_1".to_string());
+            .with_sstable(1, "gen_1".to_string());
 
         let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot])
             .with_in_memory_memtables(
@@ -2237,10 +2248,10 @@ mod tests {
             "`_rowaddr` is incompatible with vector_search's fast_search; must be NULL"
         );
 
-        // id=2 (flushed): both NULL — per-source values would collide with base.
-        let (rid_null, raddr_null) = seen.get(&2).expect("flushed row id=2 missing");
-        assert!(rid_null, "flushed row `_rowid` must be NULL");
-        assert!(raddr_null, "flushed row `_rowaddr` must be NULL");
+        // id=2 (SSTable): both NULL — per-source values would collide with base.
+        let (rid_null, raddr_null) = seen.get(&2).expect("SSTable row id=2 missing");
+        assert!(rid_null, "SSTable row `_rowid` must be NULL");
+        assert!(raddr_null, "SSTable row `_rowaddr` must be NULL");
 
         // id=3 (active): both NULL — BatchStore position is not a Lance row id.
         let (rid_null, raddr_null) = seen.get(&3).expect("active row id=3 missing");
@@ -2909,9 +2920,9 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_vector_search_flushed_superseded_by_newer_flushed() {
-        // An older flushed generation's stale row must be suppressed by a newer
-        // flushed generation (cross-flushed blocking, no base/active involved).
+    async fn test_vector_search_sstable_superseded_by_newer_sstable() {
+        // An older SSTable's stale row must be suppressed by a newer
+        // SSTable (cross-SSTable blocking, no base/active involved).
         use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
         use crate::index::DatasetIndexExt;
         use crate::index::vector::VectorIndexParams;
@@ -2946,8 +2957,8 @@ mod tests {
 
         let snapshot = ShardSnapshot::new(shard_id)
             .with_current_generation(3)
-            .with_flushed_generation(1, "gen_1".to_string())
-            .with_flushed_generation(2, "gen_2".to_string());
+            .with_sstable(1, "gen_1".to_string())
+            .with_sstable(2, "gen_2".to_string());
         let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]);
 
         let planner = LsmVectorSearchPlanner::new(
diff --git a/rust/lance/src/dataset/mem_wal/test_util.rs b/rust/lance/src/dataset/mem_wal/test_util.rs
index 43a3861e686..48e68340d3c 100644
--- a/rust/lance/src/dataset/mem_wal/test_util.rs
+++ b/rust/lance/src/dataset/mem_wal/test_util.rs
@@ -1,12 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-//! Test-only object store that injects WAL-write failures, for exercising the
-//! WAL persistence-failure fencing path.
+//! Test-only object store that injects WAL-write failures (for the WAL
+//! persistence-failure fencing path) and records the paths it serves (for
+//! asserting which opens actually resolved through a given `ObjectStoreParams`).
 
 use std::fmt::{Debug, Display, Formatter};
 use std::ops::Range;
 use std::sync::Arc;
+use std::sync::Mutex as StdMutex;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
 use bytes::Bytes;
@@ -33,6 +35,10 @@ pub struct FailControls {
     simulate_lost_ack: AtomicBool,
     /// WAL-entry `put_opts` attempts observed, for assertions.
     wal_put_attempts: AtomicUsize,
+    /// Every location written through this store.
+    put_paths: StdMutex<Vec<String>>,
+    /// Every location read through this store.
+    get_paths: StdMutex<Vec<String>>,
 }
 
 impl FailControls {
@@ -48,6 +54,26 @@ impl FailControls {
     pub fn attempts(&self) -> usize {
         self.wal_put_attempts.load(Ordering::SeqCst)
     }
+
+    /// Did any write land on a path containing `needle`? An open that resolved
+    /// its store from other params never reaches this store, so a `false` here
+    /// means the params under test did not reach that open.
+    pub fn wrote_under(&self, needle: &str) -> bool {
+        self.put_paths
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.contains(needle))
+    }
+
+    /// Did any read land on a path containing `needle`? See [`Self::wrote_under`].
+    pub fn read_under(&self, needle: &str) -> bool {
+        self.get_paths
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.contains(needle))
+    }
 }
 
 /// Wraps the inner store with [`FailingObjectStore`] at construction.
@@ -101,6 +127,11 @@ impl OSObjectStore for FailingObjectStore {
         payload: PutPayload,
         opts: PutOptions,
     ) -> OSResult<PutResult> {
+        self.controls
+            .put_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
         if Self::is_wal_entry(location) {
             self.controls
                 .wal_put_attempts
@@ -124,14 +155,30 @@ impl OSObjectStore for FailingObjectStore {
         location: &Path,
         opts: PutMultipartOptions,
     ) -> OSResult<Box<dyn MultipartUpload>> {
+        // Data files (`*.lance`) are written multipart, not via `put_opts`.
+        self.controls
+            .put_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
         self.inner.put_multipart_opts(location, opts).await
     }
 
     async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        self.controls
+            .get_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
         self.inner.get_opts(location, options).await
     }
 
     async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        self.controls
+            .get_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
         self.inner.get_ranges(location, ranges).await
     }
 
@@ -167,9 +214,11 @@ impl OSObjectStore for FailingObjectStore {
     }
 }
 
-/// Build an in-memory `ObjectStore` whose WAL-entry writes can be failed on
-/// demand. Returns the store, its base path, and the shared controls.
-pub async fn failing_memory_store() -> (Arc<ObjectStore>, Path, Arc<FailControls>) {
+/// `ObjectStoreParams` carrying the observable store wrapper, plus the controls
+/// to drive and inspect it. Open a dataset with these and every store resolved
+/// *from these params* — the base and any derived URI they are threaded to —
+/// reports its traffic back through the controls.
+pub fn observable_store_params() -> (ObjectStoreParams, Arc<FailControls>) {
     let controls = Arc::new(FailControls::default());
     let params = ObjectStoreParams {
         object_store_wrapper: Some(Arc::new(FailingWrapper {
@@ -177,6 +226,13 @@ pub async fn failing_memory_store() -> (Arc<ObjectStore>, Path, Arc<FailControls
         })),
         ..Default::default()
     };
+    (params, controls)
+}
+
+/// Build an in-memory `ObjectStore` whose WAL-entry writes can be failed on
+/// demand. Returns the store, its base path, and the shared controls.
+pub async fn failing_memory_store() -> (Arc<ObjectStore>, Path, Arc<FailControls>) {
+    let (params, controls) = observable_store_params();
     let (store, base) = ObjectStore::from_uri_and_params(
         Arc::new(ObjectStoreRegistry::default()),
         "memory:///",
diff --git a/rust/lance/src/dataset/mem_wal/util.rs b/rust/lance/src/dataset/mem_wal/util.rs
index 3f5090f6b40..dad79f54689 100644
--- a/rust/lance/src/dataset/mem_wal/util.rs
+++ b/rust/lance/src/dataset/mem_wal/util.rs
@@ -3,6 +3,7 @@
 
 //! Utility functions for MemWAL operations.
 
+use lance_io::object_store::ObjectStoreParams;
 use object_store::path::Path;
 use uuid::Uuid;
 
@@ -129,6 +130,26 @@ pub fn parse_bit_reversed_filename(filename: &str) -> Option<u64> {
     Some(bit_reverse_u64(reversed))
 }
 
+/// Adapt the store params a base dataset was opened with for use on a URI
+/// *derived* from it (an SSTable under `_mem_wal/`).
+///
+/// The deprecated `object_store` binding pins a store to one location: given
+/// `Some((store, url))`, both `ObjectStore::from_uri_and_params` and
+/// `DatasetBuilder::build_object_store` take the path from `url` and ignore the
+/// URI they were asked to open. Carried onto a generation URI it would silently
+/// redirect the open — and, on the flush path, the write — at the base table
+/// itself. Drop it so the generation URI resolves its own store; everything
+/// else (storage options, wrapper, credentials, block size) still carries over.
+///
+/// Only the base's *own* URI may reuse the params verbatim.
+pub(crate) fn derived_store_params(params: &ObjectStoreParams) -> ObjectStoreParams {
+    #[allow(deprecated)]
+    ObjectStoreParams {
+        object_store: None,
+        ..params.clone()
+    }
+}
+
 /// Path to the MemWAL root directory.
 ///
 /// Returns: `{base_path}/_mem_wal/`
@@ -157,29 +178,24 @@ pub fn shard_manifest_path(base_path: &Path, shard_id: &Uuid) -> Path {
     shard_base_path(base_path, shard_id).join("manifest")
 }
 
-/// Path to a flushed MemTable directory.
+/// Path to an SSTable directory.
 ///
 /// Returns: `{base_path}/_mem_wal/{shard_id}/{random_hash}_gen_{generation}/`
-pub fn flushed_memtable_path(
-    base_path: &Path,
-    shard_id: &Uuid,
-    random_hash: &str,
-    generation: u64,
-) -> Path {
+pub fn sstable_path(base_path: &Path, shard_id: &Uuid, random_hash: &str, generation: u64) -> Path {
     shard_base_path(base_path, shard_id).join(format!("{}_gen_{}", random_hash, generation))
 }
 
-/// Subdirectory of a flushed generation holding its standalone primary-key
+/// Subdirectory of an SSTable holding its standalone primary-key
 /// dedup index (a sidecar BTree, not registered in the manifest). Both the
 /// flush writer and the block-list probe join this onto the generation path.
 pub const PK_INDEX_DIR: &str = "_pk_index";
 
-/// Path to a flushed generation's standalone primary-key dedup index.
+/// Path to an SSTable's standalone primary-key dedup index.
 pub fn pk_index_path(gen_path: &Path) -> Path {
     gen_path.clone().join(PK_INDEX_DIR)
 }
 
-/// Generate an 8-character random hex string for flushed MemTable directories.
+/// Generate an 8-character random hex string for SSTable directories.
 pub fn generate_random_hash() -> String {
     let bytes: [u8; 4] = rand::random();
     format!(
@@ -288,7 +304,7 @@ mod tests {
         );
 
         assert_eq!(
-            flushed_memtable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(),
+            sstable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(),
             "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/a1b2c3d4_gen_5"
         );
 
@@ -372,4 +388,40 @@ mod tests {
         drop(cell);
         assert_eq!(handle.await.unwrap(), None);
     }
+
+    /// The path-bound store binding is the only thing dropped — credentials and
+    /// storage options must still reach the generation's store.
+    #[test]
+    fn test_derived_store_params_drops_only_the_path_bound_store() {
+        let accessor = lance_io::object_store::StorageOptionsAccessor::with_static_options(
+            std::collections::HashMap::from([("access_key_id".to_string(), "key".to_string())]),
+        );
+        #[allow(deprecated)]
+        let params = ObjectStoreParams {
+            object_store: Some((
+                std::sync::Arc::new(object_store::memory::InMemory::new()),
+                url::Url::parse("memory:///base").unwrap(),
+            )),
+            block_size: Some(1234),
+            storage_options_accessor: Some(std::sync::Arc::new(accessor)),
+            ..Default::default()
+        };
+
+        let derived = derived_store_params(&params);
+
+        #[allow(deprecated)]
+        {
+            assert!(
+                derived.object_store.is_none(),
+                "a store pinned to the base path must not be reused for a generation URI"
+            );
+        }
+        assert_eq!(derived.block_size, Some(1234));
+        assert_eq!(
+            derived
+                .storage_options()
+                .and_then(|o| o.get("access_key_id")),
+            Some(&"key".to_string()),
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index b4caff180e1..68eb41a03c2 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -9,7 +9,8 @@
 use std::io::Cursor;
 use std::sync::Arc;
 use std::sync::Mutex as StdMutex;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::MutexGuard as StdMutexGuard;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::time::Instant;
 
 use std::time::Duration;
@@ -87,68 +88,200 @@ impl WalFlushFailure {
     }
 }
 
-/// Watcher for batch durability using watermark-based tracking.
+/// The writer's cursors, shared by the WAL-append task, the index-apply task,
+/// every memtable's `IndexStore`, and every `put` waiting to become visible.
 ///
-/// Uses a shared watch channel that broadcasts the durable watermark.
-/// The watcher waits until the watermark reaches or exceeds its target batch ID.
-#[derive(Clone)]
-pub struct BatchDurableWatcher {
-    /// Watch receiver for the durable watermark.
-    rx: watch::Receiver<usize>,
-    /// Target batch ID to wait for.
-    target_batch_position: usize,
-    /// Terminal flush failure shared with the flusher. When set, the watermark
-    /// can never reach the target, so `wait` returns this typed error instead of
-    /// blocking forever.
+/// Two cursors are stored, one view is derived:
+///
+/// - `durable` — writer-global count of WAL-durable batches, advanced by the
+///   WAL-append task. Exclusive; 0 means none.
+/// - `indexed` — per-memtable count of indexed batches, advanced by the
+///   index-apply task. It lives on the memtable's own `IndexStore`, not here,
+///   because each memtable has its own indexes.
+/// - `visible` — **derived, never stored**: a batch is visible once it is
+///   indexed and, under `durable_write`, also durable.
+///
+/// Deriving `visible` rather than caching it is deliberate. A cached
+/// `min(indexed, durable)` recomputed by two independent tasks is the classic
+/// store-buffer race: with `Release`/`Acquire` each task can read the other's
+/// *pre-store* value, so both compute a minimum below the true one, and a
+/// max-clamped publish then leaves the cached value permanently short. A `put`
+/// blocked on it would hang until some unrelated write happened to move a cursor
+/// again. With nothing cached there is nothing to leave stale — `notify` is a
+/// bare wake-up and every waiter recomputes from the cursors themselves.
+pub struct WriterCursors {
+    durable: AtomicUsize,
+    /// Bumped whenever a cursor advances or the writer poisons. Carries no
+    /// value; it exists only to wake waiters, which then recompute.
+    notify_tx: watch::Sender<u64>,
+    notify_rx: watch::Receiver<u64>,
+    /// First terminal failure. Shared so a poisoned writer wakes every waiter
+    /// with the typed error, instead of leaving it blocked on a cursor that can
+    /// never advance again.
     terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
+    /// Whether durability is part of visibility. Per-writer, not per-write:
+    /// `visible` is a writer-wide definition, so a mix would need two visibility
+    /// views over one memtable.
+    durable_write: bool,
+}
+
+impl WriterCursors {
+    pub fn new(durable_write: bool) -> Self {
+        let (notify_tx, notify_rx) = watch::channel(0);
+        Self {
+            durable: AtomicUsize::new(0),
+            notify_tx,
+            notify_rx,
+            terminal_error: Arc::new(StdMutex::new(None)),
+            durable_write,
+        }
+    }
+
+    /// Writer-global count of WAL-durable batches.
+    pub fn durable(&self) -> usize {
+        self.durable.load(Ordering::Acquire)
+    }
+
+    pub fn durable_write(&self) -> bool {
+        self.durable_write
+    }
+
+    /// Advance the durability cursor. Monotonic, so an out-of-order completion
+    /// can never walk it backwards.
+    pub(crate) fn advance_durable(&self, global_count: usize) {
+        self.durable.fetch_max(global_count, Ordering::AcqRel);
+        self.wake();
+    }
+
+    /// Wake every waiter so it recomputes. Called after any cursor advances, and
+    /// after the writer poisons.
+    pub(crate) fn wake(&self) {
+        self.notify_tx.send_modify(|version| *version += 1);
+    }
+
+    /// The visible prefix of one memtable, given its indexed prefix and its
+    /// writer-global coordinate.
+    pub fn visible_count(&self, indexed_count: usize, global_offset: usize) -> usize {
+        if !self.durable_write {
+            return indexed_count;
+        }
+        indexed_count.min(self.durable().saturating_sub(global_offset))
+    }
+
+    /// Lock `terminal_error`, ignoring mutex poisoning.
+    ///
+    /// A panic under this lock cannot tear the `Option` it guards: the sole
+    /// writer builds the value first and assigns it whole, so a panic mid-section
+    /// leaves the slot exactly as it was. Surfacing poison as an error instead
+    /// would mask the latched `FenceReason` behind an unrelated "mutex poisoned"
+    /// precisely when a caller needs the real reason — and would strand
+    /// `mark_terminal_failure`, which has no error to return.
+    fn lock_terminal_error(&self) -> StdMutexGuard<'_, Option<WalFlushFailure>> {
+        self.terminal_error
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+    }
+
+    pub(crate) fn check_poisoned(&self) -> Result<()> {
+        if let Some(failure) = self.lock_terminal_error().clone() {
+            return Err(failure.into_error());
+        }
+        Ok(())
+    }
+
+    pub(crate) fn mark_terminal_failure(&self, error: &Error) {
+        {
+            let mut slot = self.lock_terminal_error();
+            if slot.is_none() {
+                *slot = Some(WalFlushFailure::from_error(error));
+            }
+        }
+        // Wake waiters without advancing anything: each re-checks `terminal_error`
+        // and returns the error rather than blocking on a cursor that can no
+        // longer move.
+        self.wake();
+    }
+}
+
+/// Blocks a `put` until its batches are visible: indexed, and — in durable mode
+/// — WAL-durable too.
+///
+/// Recomputes the condition on every wake instead of comparing against a cached
+/// watermark. `notify` only says "something moved"; this decides what that means.
+pub struct BatchDurableWatcher {
+    cursors: Arc<WriterCursors>,
+    rx: watch::Receiver<u64>,
+    /// The memtable the batches landed in; its `indexed_count` is the apply
+    /// cursor being waited on. `None` in WAL-only mode, which has no indexes.
+    indexes: Option<Arc<IndexStore>>,
+    /// Local exclusive count this write needs indexed.
+    target_indexed: usize,
+    /// Writer-global exclusive count this write needs durable. Batch positions
+    /// restart at 0 in every memtable while the durability cursor spans the
+    /// writer's whole life, so the caller must globalize this.
+    target_durable: usize,
 }
 
 impl BatchDurableWatcher {
-    /// Create a new watcher for a specific batch ID.
     pub fn new(
-        rx: watch::Receiver<usize>,
-        target_batch_position: usize,
-        terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
+        cursors: Arc<WriterCursors>,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
     ) -> Self {
+        let rx = cursors.notify_rx.clone();
         Self {
+            cursors,
             rx,
-            target_batch_position,
-            terminal_error,
+            indexes,
+            target_indexed,
+            target_durable,
         }
     }
 
-    /// Wait until the batch is durable.
-    ///
-    /// Returns Ok(()) when `durable_watermark >= target_batch_position`, or
-    /// Err if a terminal flush failure (e.g. a fence) means the watermark can
-    /// never reach the target.
+    /// Whether the write is readable yet.
+    fn is_visible(&self) -> bool {
+        // WAL-only mode has no indexes, so there is nothing to index-wait on.
+        let indexed = match &self.indexes {
+            Some(indexes) => indexes.indexed_count(),
+            None => self.target_indexed,
+        };
+        if indexed < self.target_indexed {
+            return false;
+        }
+        !self.cursors.durable_write() || self.cursors.durable() >= self.target_durable
+    }
+
+    /// Wait until the write is visible, or until the writer poisons — in which
+    /// case no cursor will ever reach the target, so surface the typed error
+    /// rather than blocking forever.
     pub async fn wait(&mut self) -> Result<()> {
         loop {
-            if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
-                return Err(failure.into_error());
-            }
-            let current = *self.rx.borrow();
-            if current >= self.target_batch_position {
+            // Mark the current version seen *before* testing, so a wake-up landing
+            // between the test and `changed()` below is not lost.
+            self.rx.borrow_and_update();
+            self.cursors.check_poisoned()?;
+            if self.is_visible() {
                 return Ok(());
             }
             self.rx
                 .changed()
                 .await
-                .map_err(|_| Error::io("Durable watermark channel closed"))?;
+                .map_err(|_| Error::io("Writer cursor channel closed"))?;
         }
     }
 
-    /// Check if the batch is already durable (non-blocking).
+    /// Non-blocking check.
     pub fn is_durable(&self) -> bool {
-        *self.rx.borrow() >= self.target_batch_position
+        self.is_visible()
     }
 }
 
 impl std::fmt::Debug for BatchDurableWatcher {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("BatchDurableWatcher")
-            .field("target_batch_position", &self.target_batch_position)
-            .field("current_watermark", &*self.rx.borrow())
+            .field("target_indexed", &self.target_indexed)
+            .field("target_durable", &self.target_durable)
             .finish()
     }
 }
@@ -164,20 +297,15 @@ pub struct WalEntry {
     pub num_batches: usize,
 }
 
-/// Result of a parallel WAL flush with index update.
+/// Result of a WAL flush. Append-only: index application runs on its own task
+/// (see `apply_index_range`), which records its own stats, so this no longer
+/// carries index-update timing or row counts.
 #[derive(Debug, Clone)]
 pub struct WalFlushResult {
     /// WAL entry that was written (if any).
     pub entry: Option<WalEntry>,
     /// Duration of WAL I/O operation.
     pub wal_io_duration: std::time::Duration,
-    /// Overall wall-clock duration of the index update operation.
-    /// This includes any overhead from thread scheduling and context switching.
-    pub index_update_duration: std::time::Duration,
-    /// Per-index update durations. Key is index name, value is duration.
-    pub index_update_duration_breakdown: std::collections::HashMap<String, std::time::Duration>,
-    /// Number of rows indexed.
-    pub rows_indexed: usize,
     /// Size of WAL data written in bytes.
     pub wal_bytes: usize,
 }
@@ -185,32 +313,142 @@ pub struct WalFlushResult {
 /// Source for a WAL flush — either a `BatchStore` range (MemTable mode) or
 /// a drainable in-memory pending queue (WAL-only mode).
 pub enum WalFlushSource {
-    /// MemTable mode: read a `[max_flushed+1, end_batch_position)` range
-    /// from a `BatchStore`. Indexes are updated in parallel with the WAL
-    /// append.
-    BatchStore {
-        batch_store: Arc<BatchStore>,
-        indexes: Option<Arc<IndexStore>>,
-    },
+    /// MemTable mode: append the `[durable, end_batch_position)` range of a
+    /// `BatchStore` to the WAL. Append-only — the index apply runs on its own
+    /// task, so a failed append can no longer publish rows through it.
+    BatchStore { batch_store: Arc<BatchStore> },
+    /// Timer-driven: append whichever store still owes the WAL an append,
+    /// resolved when the message is *handled*, not when it is enqueued.
+    ///
+    /// Carries no store on purpose. `MessageFactory` is synchronous and cannot
+    /// take the async state lock, and capturing an `Arc<BatchStore>` once at
+    /// handler construction would pin the first memtable forever.
+    ///
+    /// Resolution walks the live stores **oldest first** and takes the first with
+    /// `global_end() > durable`. It must not simply take "the active memtable":
+    /// a tick enqueued before a freeze is handled *after* it, would resolve to
+    /// the new memtable, and would append its batches ahead of the outgoing
+    /// memtable's tail. WAL entry positions are assigned in append-call order,
+    /// replay walks them ascending, row positions follow, and primary-key recency
+    /// is "newest visible row position wins" — so an out-of-order append silently
+    /// inverts dedup after a crash: the stale row wins. A full scan looks fine.
+    NextPending,
     /// WAL-only mode: drain all pending batches from the shared
     /// `WalOnlyState`. There are no in-memory indexes to update.
     WalOnly { state: Arc<WalOnlyState> },
 }
 
 impl WalFlushSource {
-    fn pending_count(&self) -> usize {
+    fn kind(&self) -> &'static str {
         match self {
-            Self::BatchStore { batch_store, .. } => batch_store.pending_wal_flush_count(),
-            Self::WalOnly { state } => state
-                .pending
-                .lock()
-                .ok()
-                .map(|p| p.batches.len())
-                .unwrap_or(0),
+            Self::BatchStore { .. } => "BatchStore",
+            Self::WalOnly { .. } => "WalOnly",
+            Self::NextPending => "NextPending",
         }
     }
 }
 
+/// Message to trigger an index apply.
+///
+/// Carries the store the batches actually landed in, captured on the put path
+/// *before* any freeze can rotate the memtable — pairing a new store with an old
+/// store's end position would index the wrong range.
+#[derive(Clone)]
+pub struct TriggerIndexApply {
+    pub batch_store: Arc<BatchStore>,
+    pub indexes: Arc<IndexStore>,
+    /// Local exclusive end of the range to cover.
+    pub end_batch_position: usize,
+}
+
+impl std::fmt::Debug for TriggerIndexApply {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TriggerIndexApply")
+            .field("end_batch_position", &self.end_batch_position)
+            .finish()
+    }
+}
+
+/// What an `apply_index_range` call actually indexed. `rows_indexed == 0`
+/// marks a routine coalesced no-op (the range was already covered) that must
+/// not be recorded as an index update.
+#[derive(Debug, Default, Clone, Copy)]
+pub struct IndexApplyStats {
+    pub rows_indexed: usize,
+    /// Wall-clock time spent applying the range, including the thread-scheduling
+    /// overhead of the blocking hand-off.
+    pub duration: std::time::Duration,
+}
+
+/// Apply a contiguous range of batches to a memtable's in-memory indexes.
+///
+/// Runs on its own task, as the single sequential consumer of its own channel.
+/// Being a single consumer is what makes it safe: it guarantees in-order,
+/// contiguous ranges, which is exactly what `HnswGraph::insert_batch` requires —
+/// it hard-rejects any range whose start is not `indexed_len`. Ordering comes
+/// from the task, not from the flush interval, so triggering per-put is exactly
+/// as safe as triggering on a timer.
+///
+/// It has its own channel rather than sharing the WAL flusher's, because
+/// `TaskDispatcher::run` awaits `handle()` inline: a shared channel would put a
+/// ~100ms S3 PUT in front of every latency-sensitive index apply.
+pub async fn apply_index_range(
+    cursors: &Arc<WriterCursors>,
+    message: TriggerIndexApply,
+) -> Result<IndexApplyStats> {
+    let TriggerIndexApply {
+        batch_store,
+        indexes,
+        end_batch_position,
+    } = message;
+
+    // Self-batching: a message handled while more puts queue behind it covers
+    // everything committed so far, so the ones behind it find their range already
+    // applied. Redundant messages are therefore *routine*, not exceptional, and
+    // must be a clean no-op — never a call into `insert_batches`, whose HNSW arm
+    // hard-rejects a non-contiguous start, and which is now terminal for the
+    // writer.
+    let start = indexes.indexed_count();
+    if end_batch_position <= start {
+        return Ok(IndexApplyStats::default());
+    }
+
+    // Every position in the range must exist. `get` returns `None` only for a
+    // position past `committed_len`, so a hole means the caller asked to index a
+    // batch the store never committed. Silently skipping it would let
+    // `insert_batches` advance `indexed_count` past a never-indexed batch —
+    // rows counted visible but absent from every index. Fail loudly; the handler
+    // poisons and reopen rebuilds the indexes from the WAL.
+    let stored: Vec<StoredBatch> = (start..end_batch_position)
+        .map(|position| {
+            batch_store.get(position).cloned().ok_or_else(|| {
+                Error::internal(format!(
+                    "index apply range [{start}, {end_batch_position}) is missing batch \
+                     position {position}; batch_store committed_len is {}",
+                    batch_store.len()
+                ))
+            })
+        })
+        .collect::<Result<_>>()?;
+
+    // `insert_batches` advances `indexed_count` itself, once every index has
+    // taken the batch. Time the whole hand-off so the recorded latency reflects
+    // the blocking-pool scheduling too, matching the old inline-flush measurement.
+    let rows_indexed: usize = stored.iter().map(|b| b.num_rows).sum();
+    let apply_start = Instant::now();
+    tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
+        .await
+        .map_err(|e| Error::internal(format!("Index apply task panicked: {e}")))??;
+    let duration = apply_start.elapsed();
+
+    // Wake anything waiting to become visible.
+    cursors.wake();
+    Ok(IndexApplyStats {
+        rows_indexed,
+        duration,
+    })
+}
+
 /// Message to trigger a WAL flush.
 ///
 /// Carries a `source` describing where to read batches from (BatchStore range
@@ -218,7 +456,7 @@ impl WalFlushSource {
 pub struct TriggerWalFlush {
     pub source: WalFlushSource,
     /// End batch position (exclusive). For `BatchStore`, flush batches after
-    /// `max_flushed_batch_position` up to this. For `WalOnly`, indicates the
+    /// the writer-global durable cursor up to this. For `WalOnly`, indicates the
     /// position the durability watermark must reach for callers waiting on
     /// this flush. Use `usize::MAX` to flush all pending batches.
     pub end_batch_position: usize,
@@ -232,7 +470,7 @@ pub struct TriggerWalFlush {
 impl std::fmt::Debug for TriggerWalFlush {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("TriggerWalFlush")
-            .field("pending_batches", &self.source.pending_count())
+            .field("source", &self.source.kind())
             .field("end_batch_position", &self.end_batch_position)
             .finish()
     }
@@ -361,11 +599,9 @@ impl WalOnlyState {
 /// shared `WalAppender`. The flusher delegates the actual WAL write to the
 /// appender, optionally running a parallel index update in MemTable mode.
 pub struct WalFlusher {
-    /// Watch channel sender for durable watermark.
-    /// Broadcasts the highest batch_position that is now durable.
-    durable_watermark_tx: watch::Sender<usize>,
-    /// Watch channel receiver for creating new watchers.
-    durable_watermark_rx: watch::Receiver<usize>,
+    /// The writer's cursors. Shared with the index-apply task and every
+    /// memtable's `IndexStore`, so all three agree on what is visible.
+    cursors: Arc<WriterCursors>,
     /// Underlying WAL append primitive — owns object store, epoch, and
     /// position discovery.
     wal_appender: Arc<WalAppender>,
@@ -377,35 +613,37 @@ pub struct WalFlusher {
     /// Created at construction and recreated after each flush.
     /// Used by backpressure to wait for WAL flushes.
     wal_flush_cell: std::sync::Mutex<Option<WatchableOnceCell<super::write::DurabilityResult>>>,
-    /// First terminal flush failure, shared with every `BatchDurableWatcher`. It
-    /// wakes durability waiters (the watermark never advances) and is read by
-    /// `check_poisoned` so the write path fails fast.
-    terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
 }
 
 impl WalFlusher {
     /// Create a new WAL flusher backed by an existing `WalAppender`.
     ///
     /// The appender owns object store, epoch, and position state. The
-    /// flusher adds the durability watermark, trigger channel, and
-    /// completion cell on top.
+    /// flusher adds the trigger channel and completion cell on top, and shares
+    /// the writer's cursors.
     pub fn new(wal_appender: Arc<WalAppender>) -> Self {
+        // Defaults to durable visibility; the writer replaces this with cursors
+        // built from its own config.
+        Self::with_cursors(wal_appender, Arc::new(WriterCursors::new(true)))
+    }
+
+    pub fn with_cursors(wal_appender: Arc<WalAppender>, cursors: Arc<WriterCursors>) -> Self {
         let shard_id = wal_appender.shard_id();
-        // Initialize durable watermark at 0 (no batches durable yet)
-        let (durable_watermark_tx, durable_watermark_rx) = watch::channel(0);
-        // Create initial WAL flush cell for backpressure
         let wal_flush_cell = WatchableOnceCell::new();
         Self {
-            durable_watermark_tx,
-            durable_watermark_rx,
+            cursors,
             wal_appender,
             shard_id,
             flush_tx: None,
             wal_flush_cell: std::sync::Mutex::new(Some(wal_flush_cell)),
-            terminal_error: Arc::new(StdMutex::new(None)),
         }
     }
 
+    /// The writer's cursors, for the index-apply task and the memtables.
+    pub fn cursors(&self) -> &Arc<WriterCursors> {
+        &self.cursors
+    }
+
     /// Set the flush channel for background flush handler.
     pub fn set_flush_channel(&mut self, tx: mpsc::UnboundedSender<TriggerWalFlush>) {
         self.flush_tx = Some(tx);
@@ -420,45 +658,65 @@ impl WalFlusher {
     ///
     /// Returns a `BatchDurableWatcher` that can be awaited for durability.
     /// The actual batch data is stored in the BatchStore.
-    pub fn track_batch(&self, batch_position: usize) -> BatchDurableWatcher {
-        // Return a watcher that waits for this batch to become durable
-        // batch_position is 0-indexed, so we wait for watermark > batch_position (i.e., >= batch_position + 1)
+    /// Watch a write until it becomes visible.
+    ///
+    /// `target_indexed` is a **memtable-local** exclusive count; `target_durable`
+    /// is a **writer-global** one. They are different coordinate spaces on
+    /// purpose: batch positions restart at 0 in every memtable, while the
+    /// durability cursor spans the writer's whole life. A local durable target
+    /// would already be satisfied by a *previous* memtable's appends, and would
+    /// ack a write that never reached the WAL. Callers globalize via
+    /// `BatchStore::global_offset`.
+    pub fn track_batch(
+        &self,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
+    ) -> BatchDurableWatcher {
         BatchDurableWatcher::new(
-            self.durable_watermark_rx.clone(),
-            batch_position + 1,
-            Arc::clone(&self.terminal_error),
+            Arc::clone(&self.cursors),
+            indexes,
+            target_indexed,
+            target_durable,
         )
     }
 
-    /// Latch a terminal flush failure and wake every durability waiter (the
-    /// watermark never advances, so they must observe the error, not block).
+    /// The writer-global WAL durability cursor: how many batches of this
+    /// writer's batch sequence are durable. Exclusive count; 0 means none.
+    pub fn durable(&self) -> usize {
+        self.cursors.durable()
+    }
+
+    /// Advance the durability cursor and wake waiters.
+    pub(crate) fn advance_durable(&self, global_count: usize) {
+        self.cursors.advance_durable(global_count);
+    }
+
+    /// Latch a terminal flush failure and wake every waiter (no cursor will
+    /// advance again, so they must observe the error rather than block).
     /// Idempotent: only the first failure is retained.
     fn mark_terminal_failure(&self, error: &Error) {
-        {
-            let mut slot = self.terminal_error.lock().unwrap();
-            if slot.is_none() {
-                *slot = Some(WalFlushFailure::from_error(error));
-            }
-        }
-        // Wake `wait`ers without advancing the watermark; each re-checks
-        // `terminal_error` and returns the error.
-        self.durable_watermark_tx.send_modify(|_| {});
+        self.cursors.mark_terminal_failure(error);
+    }
+
+    /// Latch a terminal failure from outside the flush path (the index-apply
+    /// task). Same effect: reads and writes fail fast, waiters wake with the
+    /// typed error, and recovery is reopen -> replay.
+    pub(crate) fn poison(&self, error: &Error) {
+        self.cursors.mark_terminal_failure(error);
     }
 
     /// Fail fast with the typed error if this writer has been fenced (by a peer
-    /// or its own persistence failure). The write path calls this before touching
-    /// the memtable so a poisoned writer can't diverge further. Recovery is to
-    /// reopen the shard (replay the WAL).
+    /// or its own persistence failure). Both the read and write paths call this
+    /// so a poisoned writer can neither diverge further nor serve a snapshot
+    /// that replay will not reproduce. Recovery is to reopen and replay.
     pub fn check_poisoned(&self) -> Result<()> {
-        if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
-            return Err(failure.into_error());
-        }
-        Ok(())
+        self.cursors.check_poisoned()
     }
 
     /// Get the current durable watermark.
     pub fn durable_watermark(&self) -> usize {
-        *self.durable_watermark_rx.borrow()
+        self.cursors.durable()
     }
 
     /// Get a watcher for WAL flush completion.
@@ -527,14 +785,16 @@ impl WalFlusher {
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
         let result = match source {
-            WalFlushSource::BatchStore {
-                batch_store,
-                indexes,
-            } => {
-                self.flush_from_batch_store(batch_store, indexes.clone(), end_batch_position)
+            WalFlushSource::BatchStore { batch_store } => {
+                self.flush_from_batch_store(batch_store, end_batch_position)
                     .await
             }
             WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await,
+            // The handler resolves a tick to a concrete store before it gets
+            // here, because only it can take the async state lock.
+            WalFlushSource::NextPending => Err(Error::internal(
+                "WalFlushSource::NextPending must be resolved by the flush handler",
+            )),
         };
         // A terminal failure means the watermark can never advance; latch the
         // poison so waiters wake with the typed error and later writes fail fast.
@@ -546,80 +806,64 @@ impl WalFlusher {
         result
     }
 
+    /// Append this store's un-appended suffix to the WAL. **Append-only.**
+    ///
+    /// The index apply used to run here, concurrently, under a `tokio::join!`.
+    /// That was the source of the dirty read: `join!` runs both arms to
+    /// completion and does not cancel the index arm when the append fails, so a
+    /// failed append still advanced the cursor readers keyed off. The two
+    /// operations have nothing in common — one is an in-memory microsecond write,
+    /// the other a ~100ms S3 PUT billed per call — so they now run on separate
+    /// tasks with separate cursors, and this one only ever touches the WAL.
     async fn flush_from_batch_store(
         &self,
         batch_store: &BatchStore,
-        indexes: Option<Arc<IndexStore>>,
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
-        // Get current flush position from per-memtable watermark (inclusive)
-        // start_batch_position is the first batch to flush
-        let start_batch_position = batch_store
-            .max_flushed_batch_position()
-            .map(|w| w + 1)
-            .unwrap_or(0);
-
-        // If we've already flushed past this end, nothing to do
+        // Where this store's un-appended suffix begins, derived from the
+        // writer-global durability cursor. `local_end` clamps a cursor that
+        // predates this memtable to 0 and one past its end to `committed_len`.
+        let start_batch_position = batch_store.local_end(self.durable());
+
+        // Already appended past this end: nothing to do. Redundant triggers are
+        // routine (a put and the freeze can both target the same range), so this
+        // is the common case, not an error.
         if start_batch_position >= end_batch_position {
             return Ok(empty_flush_result());
         }
 
-        // Collect batches in range [start_batch_position, end_batch_position)
-        let mut stored_batches: Vec<StoredBatch> =
-            Vec::with_capacity(end_batch_position - start_batch_position);
-
-        for batch_position in start_batch_position..end_batch_position {
-            if let Some(stored) = batch_store.get(batch_position) {
-                stored_batches.push(stored.clone());
-            }
-        }
-
-        if stored_batches.is_empty() {
-            return Ok(empty_flush_result());
-        }
+        // Every position in the range must exist. `get` returns `None` only for a
+        // position past `committed_len`, so a hole means we were asked to append a
+        // batch the store never committed. Silently skipping it while still
+        // advancing durability to `end_batch_position` (below) would mark an
+        // un-appended batch durable and lose it on replay — a divergence that
+        // survives the crash and hides from a full scan. Return a terminal error
+        // so `flush` poisons the writer; reopen replays the WAL.
+        let stored_batches: Vec<StoredBatch> = (start_batch_position..end_batch_position)
+            .map(|batch_position| {
+                batch_store.get(batch_position).cloned().ok_or_else(|| {
+                    Error::writer_poisoned(format!(
+                        "WAL flush range [{start_batch_position}, {end_batch_position}) is \
+                         missing batch position {batch_position}; batch_store committed_len is {}",
+                        batch_store.len()
+                    ))
+                })
+            })
+            .collect::<Result<_>>()?;
 
-        let rows_to_index: usize = stored_batches.iter().map(|b| b.num_rows).sum();
         let record_batches: Vec<RecordBatch> =
             stored_batches.iter().map(|s| s.data.clone()).collect();
 
-        let appender = self.wal_appender.clone();
-        let (append_result, index_result) = if let Some(idx_registry) = indexes {
-            let wal_future = async move {
-                let start = Instant::now();
-                let r = appender.append(record_batches).await?;
-                Ok::<_, Error>((r, start.elapsed()))
-            };
-            let index_future = async {
-                let start = Instant::now();
-                let per_index = tokio::task::spawn_blocking(move || {
-                    idx_registry.insert_batches_parallel(&stored_batches)
-                })
-                .await
-                .map_err(|e| Error::internal(format!("Index update task panicked: {}", e)))??;
-                Ok::<_, Error>((start.elapsed(), per_index))
-            };
-            tokio::join!(wal_future, index_future)
-        } else {
-            let wal_future = async move {
-                let start = Instant::now();
-                let r = appender.append(record_batches).await?;
-                Ok::<_, Error>((r, start.elapsed()))
-            };
-            (
-                wal_future.await,
-                Ok((std::time::Duration::ZERO, std::collections::HashMap::new())),
-            )
-        };
-
-        let (append_result, wal_io_duration) = append_result?;
-        let (index_update_duration, index_update_duration_breakdown) = index_result?;
-
-        // Update per-memtable watermark (inclusive: last batch ID that was flushed)
-        batch_store.set_max_flushed_batch_position(end_batch_position - 1);
+        let start = Instant::now();
+        let append_result = self.wal_appender.append(record_batches).await?;
+        let wal_io_duration = start.elapsed();
 
-        // Notify durability waiters (global channel)
-        let _ = self.durable_watermark_tx.send(end_batch_position);
-        // Signal WAL flush completion for backpressure waiters
+        // Advance the writer-global durability cursor and wake waiters. The range
+        // just appended is `[start, end)` *local to this store*, so it must be
+        // lifted into the writer's coordinate space before it is published —
+        // otherwise a fresh memtable's small local end would be compared against a
+        // cursor carrying a previous memtable's larger one.
+        self.advance_durable(batch_store.global_offset() + end_batch_position);
         self.signal_wal_flush_complete();
 
         Ok(WalFlushResult {
@@ -629,9 +873,6 @@ impl WalFlusher {
                 num_batches: append_result.num_batches,
             }),
             wal_io_duration,
-            index_update_duration,
-            index_update_duration_breakdown,
-            rows_indexed: rows_to_index,
             wal_bytes: append_result.wal_bytes,
         })
     }
@@ -649,13 +890,15 @@ impl WalFlusher {
         let append_result = self.wal_appender.append(snapshot.batches).await?;
         let wal_io_duration = start.elapsed();
 
-        // Append succeeded — remove the flushed batches from the front of
-        // the queue. Note: WAL-only mode does not use the global durability
-        // watermark (`durable_watermark_tx`) — `put_wal_only` waits on the
-        // per-trigger `done` cell instead — so we don't advance it here.
-        // Same for the wal-flush-completion cell, which is only consulted
-        // by MemTable-mode backpressure waiters.
+        // Append succeeded — remove the flushed batches from the front of the
+        // queue, then advance the writer-global durability cursor so durable
+        // `put_wal_only` callers waiting on their `BatchDurableWatcher` wake.
+        // The queue is strict FIFO with contiguous positions and its front
+        // always sits at the current watermark, so the newly-durable suffix
+        // ends at `durable + count`. Flushes are serialized on the single
+        // handler task, so this read-then-advance cannot race another flush.
         state.commit_flushed(snapshot.count);
+        self.advance_durable(self.durable() + snapshot.count);
 
         Ok(WalFlushResult {
             entry: Some(WalEntry {
@@ -664,9 +907,6 @@ impl WalFlusher {
                 num_batches: append_result.num_batches,
             }),
             wal_io_duration,
-            index_update_duration: std::time::Duration::ZERO,
-            index_update_duration_breakdown: std::collections::HashMap::new(),
-            rows_indexed: 0,
             wal_bytes: append_result.wal_bytes,
         })
     }
@@ -699,9 +939,6 @@ pub fn empty_flush_result() -> WalFlushResult {
     WalFlushResult {
         entry: None,
         wal_io_duration: std::time::Duration::ZERO,
-        index_update_duration: std::time::Duration::ZERO,
-        index_update_duration_breakdown: std::collections::HashMap::new(),
-        rows_indexed: 0,
         wal_bytes: 0,
     }
 }
@@ -774,7 +1011,7 @@ impl WalEntryData {
 /// First valid WAL entry position. Positions are 1-based so that a
 /// `ShardManifest::replay_after_wal_entry_position` of 0 unambiguously means
 /// "no flush has ever stamped the cursor" — replay then starts at position 1
-/// without needing to consult `flushed_generations`, which an external
+/// without needing to consult `sstables`, which an external
 /// compactor may legitimately drain back to empty.
 const FIRST_WAL_ENTRY_POSITION: u64 = 1;
 const MAX_APPEND_CREATE_CONFLICTS: usize = 1024;
@@ -1555,18 +1792,22 @@ mod tests {
     fn batch_store_source(batch_store: &Arc<BatchStore>) -> WalFlushSource {
         WalFlushSource::BatchStore {
             batch_store: batch_store.clone(),
-            indexes: None,
         }
     }
 
-    fn batch_store_source_with_indexes(
-        batch_store: &Arc<BatchStore>,
-        indexes: &Arc<IndexStore>,
-    ) -> WalFlushSource {
-        WalFlushSource::BatchStore {
-            batch_store: batch_store.clone(),
-            indexes: Some(indexes.clone()),
-        }
+    /// Run the index-apply task's body, as the writer's index task would.
+    async fn apply_all(batch_store: &Arc<BatchStore>, indexes: &Arc<IndexStore>) -> Result<()> {
+        let cursors = Arc::new(WriterCursors::new(true));
+        apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len(),
+            },
+        )
+        .await
+        .map(|_| ())
     }
 
     #[tokio::test]
@@ -1576,7 +1817,7 @@ mod tests {
         let buffer = build_test_flusher(store, &base_path, shard_id, 1);
 
         // Track a batch
-        let watcher = buffer.track_batch(0);
+        let watcher = buffer.track_batch(None, 0, 1);
 
         // Watcher should not be durable yet
         assert!(!watcher.is_durable());
@@ -1595,7 +1836,7 @@ mod tests {
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 10)).unwrap();
 
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         // wait() must NOT resolve before the flush happens
         let result =
@@ -1628,13 +1869,13 @@ mod tests {
         batch_store.append(batch2).unwrap();
 
         // Track batch IDs in WAL flusher
-        let mut watcher1 = buffer.track_batch(0);
-        let mut watcher2 = buffer.track_batch(1);
+        let mut watcher1 = buffer.track_batch(None, 0, 1);
+        let mut watcher2 = buffer.track_batch(None, 0, 2);
 
         // Verify initial state
         assert!(!watcher1.is_durable());
         assert!(!watcher2.is_durable());
-        assert!(batch_store.max_flushed_batch_position().is_none());
+        assert_eq!(buffer.durable(), 0);
 
         // Flush all pending batches
         let source = batch_store_source(&batch_store);
@@ -1646,8 +1887,8 @@ mod tests {
         assert_eq!(entry.position, FIRST_WAL_ENTRY_POSITION);
         assert_eq!(entry.writer_epoch, 1);
         assert_eq!(entry.num_batches, 2);
-        // After flushing 2 batches (positions 0 and 1), max flushed position is 1 (inclusive)
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(1));
+        // Two batches appended => the writer-global durable count is 2 (exclusive).
+        assert_eq!(buffer.durable(), 2);
 
         // Watchers should be notified
         watcher1.wait().await.unwrap();
@@ -1658,12 +1899,19 @@ mod tests {
 
     // Regression test for the visibility-cursor bug: with an empty IndexStore
     // (the common case for WAL-managed tables that mirror an index-less base
-    // dataset), a WAL flush must still advance `max_visible_batch_position` so
-    // scanners can see every batch up to the durable position — not just
-    // batch 0. Before the fix, the cursor stayed at 0 for the lifetime of the
-    // memtable and scanners returned only the first row.
+    /// The index apply and the WAL append are separate tasks with separate
+    /// cursors. Appending makes a range durable; it does not index it. Indexing
+    /// makes it indexed; it does not make it durable. Only both together make it
+    /// visible.
+    ///
+    /// This also covers the empty-registry case (a memtable with no configured
+    /// indexes), which used to be skipped entirely by the flush's index arm and
+    /// so left the cursor stuck at 0 for the memtable's whole life.
+    #[rstest::rstest]
+    #[case::no_indexes(false)]
+    #[case::btree_index(true)]
     #[tokio::test]
-    async fn test_wal_flush_advances_visibility_with_empty_indexes() {
+    async fn test_append_and_index_advance_separate_cursors(#[case] with_btree: bool) {
         let (store, base_path, _temp_dir) = create_local_store().await;
         let shard_id = Uuid::new_v4();
         let flusher = build_test_flusher(store, &base_path, shard_id, 1);
@@ -1674,28 +1922,149 @@ mod tests {
             batch_store.append(create_test_batch(&schema, 5)).unwrap();
         }
 
-        // Empty registry, mimicking a memtable with `index_configs = []`.
-        let indexes = Arc::new(IndexStore::new());
-        assert_eq!(indexes.max_visible_batch_position(), 0);
+        let mut idx = IndexStore::new();
+        if with_btree {
+            idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        }
+        let indexes = Arc::new(idx);
 
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
-        flusher.flush(&source, batch_store.len()).await.unwrap();
+        // The append alone makes the range durable and indexes nothing.
+        flusher
+            .flush(&batch_store_source(&batch_store), batch_store.len())
+            .await
+            .unwrap();
+        assert_eq!(flusher.durable(), 3);
+        assert_eq!(indexes.indexed_count(), 0);
 
-        // Cursor must advance to the highest flushed batch position (2),
-        // making all three batches visible to scanners.
-        assert_eq!(indexes.max_visible_batch_position(), 2);
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(2));
+        // The index apply alone advances the index cursor.
+        apply_all(&batch_store, &indexes).await.unwrap();
+        assert_eq!(indexes.indexed_count(), 3);
     }
 
-    // Regression guard for the indexed path: with at least one BTree index
-    // configured, the cursor advance still fires (this was already working
-    // before the fix — keeping the test to lock in the behavior).
+    /// A WAL flush asked to cover a range past the store's committed length must
+    /// fail terminally, not silently short-append. The store is append-only, so
+    /// `get` returns `None` only past `committed_len`; skipping that position
+    /// while still advancing durability to `end_batch_position` would mark an
+    /// un-appended batch durable and lose it on replay. The flush must poison
+    /// instead, and durability must not move.
     #[tokio::test]
-    async fn test_wal_flush_advances_visibility_with_btree_index() {
+    async fn test_flush_rejects_range_past_committed_len() {
         let (store, base_path, _temp_dir) = create_local_store().await;
         let shard_id = Uuid::new_v4();
         let flusher = build_test_flusher(store, &base_path, shard_id, 1);
 
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+
+        // Two batches committed (positions 0, 1); ask to flush through position 2.
+        let err = flusher
+            .flush(&batch_store_source(&batch_store), batch_store.len() + 1)
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(
+            err.to_string().contains("missing batch position 2"),
+            "unexpected error: {err}"
+        );
+
+        // Terminal: the flusher is poisoned and durability never advanced past
+        // what was actually appended.
+        assert!(flusher.check_poisoned().is_err());
+        assert_eq!(flusher.durable(), 0);
+    }
+
+    /// The index-apply path has the same invariant: a range past the committed
+    /// length must error rather than silently under-index and advance the cursor
+    /// past a batch that was never inserted.
+    #[tokio::test]
+    async fn test_index_apply_rejects_range_past_committed_len() {
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+
+        let indexes = Arc::new(IndexStore::new());
+        let cursors = Arc::new(WriterCursors::new(true));
+
+        // One batch committed (position 0); ask to index through position 1.
+        let err = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len() + 1,
+            },
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            err.to_string().contains("missing batch position 1"),
+            "unexpected error: {err}"
+        );
+        // The cursor did not advance past the hole.
+        assert_eq!(indexes.indexed_count(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_writer_cursors_advance_and_visibility() {
+        // The durable cursor starts at zero and advances to the value passed.
+        let cursors = WriterCursors::new(true);
+        assert_eq!(cursors.durable(), 0);
+        cursors.advance_durable(5);
+        assert_eq!(cursors.durable(), 5);
+
+        // fetch_max: a lower value never walks the cursor backwards.
+        cursors.advance_durable(3);
+        assert_eq!(cursors.durable(), 5);
+
+        // durable_write = true: visibility is clamped by the writer-global durable
+        // cursor, offset by this memtable's global coordinate.
+        assert!(cursors.durable_write());
+        // global_offset 0: min(indexed = 10, durable = 5) = 5.
+        assert_eq!(cursors.visible_count(10, 0), 5);
+        // global_offset 2: min(indexed = 10, durable 5 - 2 = 3) = 3.
+        assert_eq!(cursors.visible_count(10, 2), 3);
+        // A memtable that starts past the durable cursor sees nothing.
+        assert_eq!(cursors.visible_count(10, 8), 0);
+        // Indexing, not durability, is the tighter bound here.
+        assert_eq!(cursors.visible_count(2, 0), 2);
+
+        // durable_write = false: durability is not part of visibility, so the
+        // indexed count passes through unchanged regardless of the cursor.
+        let non_durable = WriterCursors::new(false);
+        assert!(!non_durable.durable_write());
+        assert_eq!(non_durable.visible_count(7, 0), 7);
+        non_durable.advance_durable(1);
+        assert_eq!(non_durable.visible_count(7, 0), 7);
+    }
+
+    #[tokio::test]
+    async fn test_writer_cursors_advance_is_monotonic() {
+        let cursors = Arc::new(WriterCursors::new(true));
+        let mut handles = Vec::new();
+        for target in [4usize, 1, 9, 3, 7, 2] {
+            let cursors = cursors.clone();
+            handles.push(tokio::spawn(async move {
+                let before = cursors.durable();
+                cursors.advance_durable(target);
+                // An advance can only move the cursor forward, never back — even
+                // when a smaller target races a larger one.
+                assert!(cursors.durable() >= before);
+            }));
+        }
+        for handle in handles {
+            handle.await.unwrap();
+        }
+        // Whatever order the concurrent advances landed in, the cursor ends at the
+        // largest target.
+        assert_eq!(cursors.durable(), 9);
+    }
+
+    /// The happy path of the index-apply task: a valid range advances the index
+    /// cursor to its end and reports exactly the rows it covered.
+    #[tokio::test]
+    async fn test_index_apply_advances_cursor_and_counts_rows() {
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         for _ in 0..3 {
@@ -1705,12 +2074,38 @@ mod tests {
         let mut idx = IndexStore::new();
         idx.add_btree("id_idx".to_string(), 0, "id".to_string());
         let indexes = Arc::new(idx);
-
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
-        flusher.flush(&source, batch_store.len()).await.unwrap();
-
-        assert_eq!(indexes.max_visible_batch_position(), 2);
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(2));
+        let cursors = Arc::new(WriterCursors::new(true));
+
+        let stats = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len(),
+            },
+        )
+        .await
+        .unwrap();
+
+        // Three batches of five rows each were indexed, and the cursor advanced to
+        // cover them.
+        assert_eq!(indexes.indexed_count(), 3);
+        assert_eq!(stats.rows_indexed, 15);
+
+        // Re-applying the same range is a coalesced no-op: the cursor holds and no
+        // rows are recounted.
+        let repeat = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len(),
+            },
+        )
+        .await
+        .unwrap();
+        assert_eq!(indexes.indexed_count(), 3);
+        assert_eq!(repeat.rows_indexed, 0);
     }
 
     #[tokio::test]
@@ -1726,8 +2121,8 @@ mod tests {
         batch_store.append(create_test_batch(&schema, 5)).unwrap();
 
         // Track batch IDs and flush all pending batches
-        let _watcher1 = buffer.track_batch(0);
-        let _watcher2 = buffer.track_batch(1);
+        let _watcher1 = buffer.track_batch(None, 0, 1);
+        let _watcher2 = buffer.track_batch(None, 0, 2);
         let source = batch_store_source(&batch_store);
         let result = buffer.flush(&source, batch_store.len()).await.unwrap();
         let entry = result.entry.unwrap();
@@ -1905,7 +2300,7 @@ mod tests {
         // A durable put on the predecessor: stage a batch and track it.
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         // Flushing collides with the sentinel and fences. Both the flush result
         // and the watcher must report the fence — and the watcher must resolve
@@ -2067,6 +2462,132 @@ mod tests {
         );
     }
 
+    /// A failed WAL append must not make rows visible, even when the index apply
+    /// has already taken them.
+    ///
+    /// The two now run on separate tasks, so an index apply that lands while the
+    /// append is failing advances `indexed_count` — which is fine, and
+    /// unavoidable: indexes are derived state and replay rebuilds them. What must
+    /// not happen is for that to make the rows *readable*, because they are not in
+    /// the WAL and replay will not reproduce them.
+    ///
+    /// Visibility is derived, not published, so this holds by construction: with
+    /// `durable_write`, `visible = min(indexed, durable)`, and a failed append
+    /// leaves `durable` at 0.
+    #[tokio::test]
+    async fn test_failed_append_indexes_but_stays_invisible() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.fail_wal_puts(usize::MAX);
+        let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2));
+        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
+        let appender = Arc::new(WalAppender::with_claimed_epoch(
+            store,
+            base,
+            shard_id,
+            manifest_store,
+            epoch,
+            0,
+            WalRetryConfig {
+                max_retries: 1,
+                base_delay: Duration::from_millis(1),
+            },
+        ));
+        let cursors = Arc::new(WriterCursors::new(true));
+        let flusher = WalFlusher::with_cursors(appender, Arc::clone(&cursors));
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+
+        let mut idx = IndexStore::new();
+        idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        idx.set_durability(Arc::clone(&cursors), 0);
+        let indexes = Arc::new(idx);
+
+        // The index apply succeeds.
+        apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: 1,
+            },
+        )
+        .await
+        .unwrap();
+        assert_eq!(indexes.indexed_count(), 1);
+
+        // The append does not.
+        let err = flusher
+            .flush(&batch_store_source(&batch_store), batch_store.len())
+            .await
+            .expect_err("the WAL PUT is failing, so the append must fail");
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // Indexed, but not durable — so not visible.
+        assert_eq!(flusher.durable(), 0);
+        assert_eq!(
+            indexes.visible_count(),
+            0,
+            "a row whose WAL append failed must never become readable"
+        );
+    }
+
+    /// An index-apply failure poisons the writer.
+    ///
+    /// A partial apply cannot be rolled back — `insert_batches` joins every index
+    /// thread unconditionally, so a failure leaves the others fully applied, and
+    /// none of HNSW, FTS or BTree has a delete. Continuing would re-cover the
+    /// range on the next attempt and corrupt the indexes that *did* succeed. So
+    /// the failure is terminal: reads and writes fail fast, and recovery is
+    /// reopen -> replay, which rebuilds the indexes from the WAL.
+    #[tokio::test]
+    async fn test_index_failure_poisons_the_writer() {
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let flusher = build_test_flusher(store, &base_path, shard_id, 1);
+        let cursors = Arc::clone(flusher.cursors());
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+
+        // An HNSW index on `id`, which is an Int32 and not a vector, so every
+        // insert of this batch fails deterministically. `validate_index_configs`
+        // rejects this at shard open — that is what makes poison-and-replay
+        // terminating — so the store has to be built by hand to reach it at all.
+        let mut idx = IndexStore::new();
+        idx.add_hnsw(
+            "bad_hnsw".to_string(),
+            0,
+            "id".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            128,
+            8,
+        );
+        let indexes = Arc::new(idx);
+
+        let err = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: 1,
+            },
+        )
+        .await
+        .expect_err("indexing an Int32 column as a vector must fail");
+
+        // The index task latches it, exactly as `IndexApplyHandler` does.
+        flusher.poison(&err);
+        assert!(
+            flusher.check_poisoned().is_err(),
+            "the writer must be poisoned rather than limp on with a corrupt index"
+        );
+        assert_eq!(indexes.visible_count(), 0);
+    }
+
     // A persistence failure during flush latches the poison: the flush result,
     // `check_poisoned`, and the durability watcher all report the typed error
     // (rather than the watcher hanging on a watermark that never advances).
@@ -2094,7 +2615,7 @@ mod tests {
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         let source = batch_store_source(&batch_store);
         let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err();
@@ -2114,4 +2635,33 @@ mod tests {
             .expect_err("watcher must surface the poison");
         assert_eq!(waited.fence_reason(), Some(FenceReason::PersistenceFailure));
     }
+
+    // A panic under the `terminal_error` lock must not cost the writer its
+    // latched failure. Recovery is reopen -> replay, which is driven by the real
+    // `FenceReason`; reporting the mutex poisoning instead would bury it.
+    #[tokio::test]
+    async fn test_poisoned_terminal_error_mutex_still_reports_typed_failure() {
+        let cursors = Arc::new(WriterCursors::new(true));
+        cursors.mark_terminal_failure(&Error::writer_poisoned("injected persistence failure"));
+
+        let terminal_error = Arc::clone(&cursors.terminal_error);
+        let panicked = std::thread::spawn(move || {
+            let _guard = terminal_error.lock().unwrap();
+            panic!("poison the terminal error mutex");
+        })
+        .join();
+        assert!(panicked.is_err());
+        assert!(cursors.terminal_error.is_poisoned());
+
+        let error = cursors.check_poisoned().unwrap_err();
+        assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(error.to_string().contains("injected persistence failure"));
+
+        // The latch still takes writes, and still keeps the first failure.
+        cursors.mark_terminal_failure(&Error::fenced_by_peer("later peer fence"));
+        assert_eq!(
+            cursors.check_poisoned().unwrap_err().fence_reason(),
+            Some(FenceReason::PersistenceFailure)
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 1b505354813..7f05cbfff3e 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -25,7 +25,7 @@ use lance_core::datatypes::Schema;
 use lance_core::{Error, Result};
 use lance_index::mem_wal::ShardManifest;
 use lance_index::vector::hnsw::builder::HnswBuildParams;
-use lance_io::object_store::ObjectStore;
+use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use log::{debug, error, info, warn};
 use object_store::path::Path;
 use tokio::sync::{RwLock, mpsc};
@@ -37,6 +37,7 @@ use uuid::Uuid;
 
 pub use super::index::{
     BTreeIndexConfig, BTreeMemIndex, FtsIndexConfig, HnswIndexConfig, IndexStore, MemIndexConfig,
+    validate_index_configs,
 };
 pub use super::memtable::CacheConfig;
 pub use super::memtable::MemTable;
@@ -47,12 +48,13 @@ pub use super::util::{WatchableOnceCell, WatchableOnceCellReader};
 pub use super::wal::{WalEntry, WalEntryData, WalFlushFailure, WalFlushResult, WalFlusher};
 
 use super::memtable::flush::TriggerMemTableFlush;
-use super::scanner::GenerationWarmer;
+use super::scanner::SsTableWarmer;
 use super::wal::{
-    BatchDurableWatcher, TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState,
-    WalRetryConfig, WalTailer, empty_flush_result,
+    BatchDurableWatcher, TriggerIndexApply, TriggerWalFlush, WalAppender, WalFlushSource,
+    WalOnlyState, WalRetryConfig, WalTailer, WriterCursors, apply_index_range, empty_flush_result,
 };
 use super::{TOMBSTONE, schema_with_tombstone};
+use crate::session::Session;
 
 use super::manifest::ShardManifestStore;
 
@@ -83,17 +85,6 @@ pub struct ShardWriterConfig {
     /// - Lower latency, batched S3 operations
     pub durable_write: bool,
 
-    /// Whether to update indexes synchronously on each write.
-    ///
-    /// When true:
-    /// - Newly written data is immediately searchable via indexes
-    /// - Higher latency due to index update overhead
-    ///
-    /// When false:
-    /// - Index updates are deferred
-    /// - New data may not appear in index-accelerated queries immediately
-    pub sync_indexed_write: bool,
-
     /// Maximum WAL buffer size in bytes before triggering a flush.
     ///
     /// This is a soft threshold - write batches are atomic and won't be split.
@@ -165,23 +156,6 @@ pub struct ShardWriterConfig {
     /// Default: 30 seconds
     pub backpressure_log_interval: Duration,
 
-    /// Maximum rows to buffer before flushing to async indexes.
-    ///
-    /// Only applies when `sync_indexed_write` is false. Larger values enable
-    /// better vectorization but increase memory usage and latency before data
-    /// becomes searchable.
-    ///
-    /// Default: 10,000 rows
-    pub async_index_buffer_rows: usize,
-
-    /// Maximum time to buffer before flushing to async indexes.
-    ///
-    /// Only applies when `sync_indexed_write` is false. Ensures bounded latency
-    /// for data to become searchable even during low write throughput.
-    ///
-    /// Default: 1 second
-    pub async_index_interval: Duration,
-
     /// Interval for periodic stats logging.
     ///
     /// Stats (write throughput, backpressure events, memtable size) are logged
@@ -191,14 +165,14 @@ pub struct ShardWriterConfig {
     pub stats_log_interval: Option<Duration>,
 
     /// How long a frozen memtable lingers in memory after its flush commits,
-    /// before it is evicted and served only from the on-disk flushed dataset.
+    /// before it is evicted and served only from the on-disk SSTable dataset.
     ///
     /// `Duration::ZERO` (the default) disables retention: evict on commit, no
     /// sweep ticker. Correct for single-shot queries, which can't observe a
     /// generation evicted mid-read.
     ///
     /// A non-zero value is required only for queries split across reads (e.g.
-    /// fresh tier and base table read separately, then deduped): the flushed
+    /// fresh tier and base table read separately, then deduped): the SSTable
     /// dataset loses the per-batch boundaries that bound as-of membership
     /// (see [`crate::dataset::mem_wal::scanner::FreshTierWatermark`]), so a
     /// generation evicted between a query's reads can serve a stale row. Set it
@@ -223,8 +197,7 @@ pub struct ShardWriterConfig {
     ///   `durable_write` settings as MemTable mode.
     ///
     /// MemTable-tied tunables (`max_memtable_size`, `max_memtable_rows`,
-    /// `max_memtable_batches`, `sync_indexed_write`, `async_index_buffer_rows`,
-    /// `async_index_interval`) are ignored when `enable_memtable == false`.
+    /// `max_memtable_batches`) are ignored when `enable_memtable == false`.
     ///
     /// For raw single-entry synchronous atomic appends with no buffering and
     /// no background tasks, use `WalAppender` directly — it is a strictly
@@ -237,7 +210,7 @@ pub struct ShardWriterConfig {
     /// These control the in-memory HNSW graph this writer builds for its
     /// MemTable (and, on flush, the on-disk graph serialized from it). They are
     /// a property of the writer that builds the MemTable, not of the index
-    /// definition: each flushed generation is independent, so different writers
+    /// definition: each SSTable is independent, so different writers
     /// may use different parameters. An index without an entry uses the default
     /// build parameters. `num_edges` is the HNSW graph degree (level 0 retains
     /// `2 * num_edges`), equivalent to FAISS's `M`.
@@ -248,7 +221,18 @@ pub struct ShardWriterConfig {
     /// Optional warmer fired pre-commit for each new generation (zero cold reads
     /// on first query). Wired to the flusher; supplied by the consumer (e.g. the
     /// WAL pod). Default: `None`.
-    pub warmer: Option<Arc<dyn GenerationWarmer>>,
+    pub warmer: Option<Arc<dyn SsTableWarmer>>,
+
+    /// Store params the base dataset was opened with, reused for the flusher's
+    /// opens + writes (base + generations). Injected by `mem_wal_writer`; set
+    /// these to the params of the dataset at `base_uri`, not to params bound to
+    /// some other path — generation URIs are derived from them.
+    /// Default: `None` (open by URI alone).
+    pub store_params: Option<ObjectStoreParams>,
+
+    /// Session for those opens, injected alongside `store_params`.
+    /// Default: `None`.
+    pub session: Option<Arc<Session>>,
 }
 
 impl Default for ShardWriterConfig {
@@ -257,7 +241,6 @@ impl Default for ShardWriterConfig {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: true,
             max_wal_buffer_size: 10 * 1024 * 1024, // 10MB
             max_wal_flush_interval: Some(Duration::from_millis(100)), // 100ms
             max_wal_persist_retries: 3,
@@ -268,13 +251,13 @@ impl Default for ShardWriterConfig {
             manifest_scan_batch_size: 2,
             max_unflushed_memtable_bytes: 1024 * 1024 * 1024, // 1GB
             backpressure_log_interval: Duration::from_secs(30),
-            async_index_buffer_rows: 10_000,
-            async_index_interval: Duration::from_secs(1),
             stats_log_interval: Some(Duration::from_secs(60)), // 1 minute
             frozen_memtable_grace: Duration::ZERO,
             enable_memtable: true,
             hnsw_params: HashMap::new(),
             warmer: None,
+            store_params: None,
+            session: None,
         }
     }
 }
@@ -300,12 +283,6 @@ impl ShardWriterConfig {
         self
     }
 
-    /// Set indexed writes mode.
-    pub fn with_sync_indexed_write(mut self, indexed: bool) -> Self {
-        self.sync_indexed_write = indexed;
-        self
-    }
-
     /// Set maximum WAL buffer size.
     pub fn with_max_wal_buffer_size(mut self, size: usize) -> Self {
         self.max_wal_buffer_size = size;
@@ -368,25 +345,13 @@ impl ShardWriterConfig {
         self
     }
 
-    /// Set async index buffer rows.
-    pub fn with_async_index_buffer_rows(mut self, rows: usize) -> Self {
-        self.async_index_buffer_rows = rows;
-        self
-    }
-
-    /// Set async index interval.
-    pub fn with_async_index_interval(mut self, interval: Duration) -> Self {
-        self.async_index_interval = interval;
-        self
-    }
-
     /// Set stats logging interval. Use None to disable periodic stats logging.
     pub fn with_stats_log_interval(mut self, interval: Option<Duration>) -> Self {
         self.stats_log_interval = interval;
         self
     }
 
-    /// Set how long a flushed memtable lingers in memory before eviction. MUST
+    /// Set how long an SSTable lingers in memory before eviction. MUST
     /// exceed the maximum query elapsed time — see `frozen_memtable_grace`.
     pub fn with_frozen_memtable_grace(mut self, grace: Duration) -> Self {
         self.frozen_memtable_grace = grace;
@@ -453,7 +418,13 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
         let mut ticker_intervals: Vec<(Interval, MessageFactory<T>)> = tickers
             .into_iter()
             .map(|(duration, factory)| {
-                let interval = interval_at(tokio::time::Instant::now() + duration, duration);
+                let mut interval = interval_at(tokio::time::Instant::now() + duration, duration);
+                // `Burst` (the default) replays every tick missed while `handle()`
+                // was running. A WAL append can easily outlast its own interval, so
+                // the missed ticks pile up, the ticker arm below is always ready,
+                // and — being `biased` — it starves `rx` indefinitely: freeze
+                // completion cells and `close()`'s final append never get handled.
+                interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
                 (interval, factory)
             })
             .collect();
@@ -496,12 +467,12 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
                         debug!("Task '{}' received cancellation", self.name);
                         break Ok(());
                     }
-                    _ = first_interval.tick() => {
-                        let message = (ticker_intervals[0].1)();
-                        if let Err(e) = self.handler.handle(message).await {
-                            error!("Task '{}' error handling ticker message: {}", self.name, e);
-                        }
-                    }
+                    // Explicit messages outrank the ticker. A tick is a backstop
+                    // — it only ever *adds* an append that a real trigger would
+                    // have made anyway — whereas a message may be a freeze's
+                    // completion cell or `close()`'s final append, which nothing
+                    // else will deliver. Polling the ticker first (as this did)
+                    // lets a ready tick starve them.
                     msg = self.rx.recv() => {
                         match msg {
                             Some(message) => {
@@ -515,6 +486,12 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
                             }
                         }
                     }
+                    _ = first_interval.tick() => {
+                        let message = (ticker_intervals[0].1)();
+                        if let Err(e) = self.handler.handle(message).await {
+                            error!("Task '{}' error handling ticker message: {}", self.name, e);
+                        }
+                    }
                 }
             }
         };
@@ -559,20 +536,39 @@ impl TaskExecutor {
         Ok(())
     }
 
+    /// Cancel and join every handler registered by [`Self::add_handler`].
+    ///
+    /// Cancellation causes each handler's dispatcher to stop accepting messages and call
+    /// [`MessageHandler::cleanup`]. This method waits for every dispatcher to finish, even if
+    /// cleanup fails or a dispatcher panics, and then returns the first such failure. It returns
+    /// `Ok(())` only after every registered handler has been cleaned up successfully.
     pub async fn shutdown_all(&self) -> Result<()> {
         info!("Shutting down all tasks");
         self.cancellation_token.cancel();
 
         let tasks = std::mem::take(&mut *self.tasks.write().unwrap());
+        let mut first_error = None;
         for (name, handle) in tasks {
             match handle.await {
                 Ok(Ok(())) => debug!("Task '{}' completed successfully", name),
-                Ok(Err(e)) => warn!("Task '{}' completed with error: {}", name, e),
-                Err(e) => error!("Task '{}' panicked: {}", name, e),
+                Ok(Err(e)) => {
+                    warn!("Task '{}' completed with error: {}", name, e);
+                    if first_error.is_none() {
+                        first_error = Some(e);
+                    }
+                }
+                Err(e) => {
+                    error!("Task '{}' panicked: {}", name, e);
+                    if first_error.is_none() {
+                        first_error = Some(Error::internal(format!(
+                            "Task '{name}' panicked during shutdown: {e}"
+                        )));
+                    }
+                }
             }
         }
 
-        Ok(())
+        first_error.map_or(Ok(()), Err)
     }
 }
 
@@ -822,7 +818,7 @@ fn now_millis() -> u64 {
     start_time().elapsed().as_millis() as u64
 }
 
-/// Replay WAL entries written after the last successfully-flushed generation
+/// Replay WAL entries written after the last successfully-flushed SSTable
 /// into the freshly-built MemTable. Updates any in-memory indexes attached to
 /// the MemTable so replayed rows are immediately searchable.
 ///
@@ -836,32 +832,61 @@ fn now_millis() -> u64 {
 /// Aborts with an error if any replayed entry's `writer_epoch` is strictly
 /// greater than `our_epoch` — that indicates a successor writer claimed the
 /// shard between our `claim_epoch` and this replay, fencing us.
+/// Outcome of replaying a shard's WAL into memory.
+struct ReplayResult {
+    /// The active memtable — the final, partial one replay left unsealed. A fresh
+    /// shard yields an empty one; every sealed memtable was flushed to a Lance
+    /// generation during replay and is not returned.
+    active: MemTable,
+    /// One past the highest WAL entry position observed — the next write position.
+    next_wal_position: u64,
+}
+
+/// Replay a shard's WAL into memory, flushing sealed memtables as the batch store
+/// fills.
+///
+/// A single memtable holds at most `max_memtable_batches` batches, but a WAL is
+/// unbounded — so replay must rotate exactly as the live write path does. It
+/// seals a full memtable and, because the data is already durable, flushes it to
+/// a Lance generation right here (the same `MemTableFlusher::flush` the live path
+/// uses), rather than holding every sealed memtable in memory until open
+/// finishes. That bounds resident memory to ~two memtables and truncates the WAL
+/// as it goes, so a later reopen replays only the unflushed tail. Only the final
+/// partial memtable is returned, as the active one.
+///
+/// `make_memtable(generation, global_offset)` builds a fresh, cursor-bound
+/// memtable. Rotation happens at WAL-entry boundaries, never mid-entry, so each
+/// sealed memtable covers a clean range of complete entries and stamps the last
+/// one as its SSTable's `replay_after_wal_entry_position`.
+#[allow(clippy::too_many_arguments)]
 async fn replay_memtable_from_wal(
     object_store: Arc<ObjectStore>,
     base_path: Path,
     shard_id: Uuid,
     our_epoch: u64,
     manifest: &ShardManifest,
-    memtable: &mut MemTable,
-) -> Result<u64> {
+    base_generation: u64,
+    mut make_memtable: impl FnMut(u64, usize) -> Result<MemTable>,
+    flusher: &MemTableFlusher,
+    wal_flusher: &WalFlusher,
+    index_configs: &[MemIndexConfig],
+    max_memtable_size: usize,
+) -> Result<ReplayResult> {
     // WAL positions are 1-based (see `FIRST_WAL_ENTRY_POSITION`), so a
     // cursor of 0 means "no flush has ever stamped this shard" and replay
     // starts at position 1. After flushing position N the cursor holds N
     // and replay starts at N+1. The arithmetic collapses to a single
     // saturating_add(1) in both cases — we deliberately do not consult
-    // `flushed_generations` here, since an external compactor may
+    // `sstables` here, since an external compactor may
     // legitimately drain that vector back to empty after merging its
     // contents into the base table.
     let start_position = manifest.replay_after_wal_entry_position.saturating_add(1);
 
-    // The MemTable is always freshly built before this function runs, so
-    // any existing BatchStore entries can only have come from this replay
-    // pass. We index everything in `[0, batch_count)` at the end.
-    debug_assert_eq!(memtable.batch_count(), 0);
-
     let tailer = WalTailer::new(object_store, base_path, shard_id);
     let mut position = start_position;
 
+    let mut active = make_memtable(base_generation, 0)?;
+
     loop {
         match tailer.read_entry(position).await? {
             // The first NotFound proves the WAL tip is at `position`, which
@@ -880,13 +905,60 @@ async fn replay_memtable_from_wal(
                     // Entries written before deletes existed lack `_tombstone`;
                     // inject `false` so they match the extended memtable schema.
                     // Normal entries already carry it and pass through unchanged.
-                    let target_schema = memtable.schema().clone();
+                    let target_schema = active.schema().clone();
                     let batches = entry
                         .batches
                         .into_iter()
                         .map(|b| ensure_tombstone_column(b, &target_schema))
                         .collect::<Result<Vec<_>>>()?;
-                    memtable.insert_batches_only(batches).await?;
+
+                    // Seal + flush at the entry boundary on the *same* criteria the
+                    // live path uses (`maybe_trigger_memtable_flush`): the memtable
+                    // is at or over `max_memtable_size` bytes, or this whole entry
+                    // won't fit the batch store. The byte trigger is the one that
+                    // matters beyond avoiding overflow — it is what keeps a memtable
+                    // under `max_memtable_rows`, and therefore keeps the in-memory
+                    // HNSW index (sized to `max_memtable_rows`) from exhausting its
+                    // capacity when the final active memtable is indexed.
+                    //
+                    // Rotate at the entry boundary so no entry is split across two
+                    // memtables and each sealed one covers a clean range of complete
+                    // entries. Never rotate an empty memtable — if a single entry
+                    // has more batches than a memtable can hold, a fresh one would
+                    // overflow too, the same hard limit the live put path has, left
+                    // to the insert below to surface.
+                    if !active.batch_store().is_empty()
+                        && memtable_reached_flush_threshold(
+                            &active,
+                            max_memtable_size,
+                            batches.len(),
+                        )
+                    {
+                        let store = active.batch_store();
+                        // The last entry this memtable fully absorbed is the one
+                        // before the entry about to be inserted.
+                        let covered = position.saturating_sub(1);
+                        let generation = active.generation() + 1;
+                        let global_end = store.global_end();
+
+                        // The sealed data is already durable in the WAL — mark it
+                        // so the flush's `all_flushed_to_wal` precondition holds and
+                        // no WAL re-append is attempted.
+                        wal_flusher.advance_durable(global_end);
+                        flush_replayed_memtable(
+                            flusher,
+                            &active,
+                            our_epoch,
+                            covered,
+                            global_end,
+                            index_configs,
+                        )
+                        .await?;
+
+                        active = make_memtable(generation, global_end)?;
+                    }
+
+                    active.insert_batches_only(batches).await?;
                 }
                 position = position.checked_add(1).ok_or_else(|| {
                     Error::io(format!(
@@ -898,21 +970,18 @@ async fn replay_memtable_from_wal(
         }
     }
 
-    // Update in-memory indexes with the replayed batches so readers see them
-    // through the index path (matching what would have happened on the
-    // pre-crash writer's WAL flush). Indexes from the previous writer don't
-    // persist; this rebuilds them from the WAL.
-    if let Some(indexes) = memtable.indexes_arc() {
-        let batches_after = memtable.batch_count();
-        if batches_after > 0 {
-            let store = memtable.batch_store();
-            let mut stored: Vec<StoredBatch> = Vec::with_capacity(batches_after);
-            for pos in 0..batches_after {
-                if let Some(s) = store.get(pos) {
-                    stored.push(s.clone());
-                }
-            }
-            tokio::task::spawn_blocking(move || indexes.insert_batches_parallel(&stored))
+    // Rebuild the active memtable's in-memory indexes from the batches just
+    // replayed, so readers see them through the index path — matching what the
+    // pre-crash writer's flush would have done. Sealed memtables needed no
+    // in-memory index build: they were flushed straight to disk and are gone.
+    if let Some(indexes) = active.indexes_arc() {
+        let batch_count = active.batch_count();
+        if batch_count > 0 {
+            let store = active.batch_store();
+            let stored: Vec<StoredBatch> = (0..batch_count)
+                .filter_map(|pos| store.get(pos).cloned())
+                .collect();
+            tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
                 .await
                 .map_err(|e| {
                     Error::internal(format!("WAL replay index update task panicked: {}", e))
@@ -920,7 +989,50 @@ async fn replay_memtable_from_wal(
         }
     }
 
-    Ok(position)
+    Ok(ReplayResult {
+        active,
+        next_wal_position: position,
+    })
+}
+
+/// Whether a memtable has reached the threshold at which it should be sealed and
+/// flushed: at or over `max_memtable_size` bytes, or without room in its batch
+/// store for `incoming_batches` more.
+///
+/// The single source of truth for the flush trigger, shared by the live put path
+/// (`maybe_trigger_memtable_flush`, checking post-insert with `incoming_batches =
+/// 1` — "is there room for the next batch") and by replay (checking pre-insert
+/// with the next WAL entry's batch count). Keeping one predicate is what stops the
+/// two from drifting — e.g. someone adding a third criterion to one and not the
+/// other, which is the exact class of bug this whole change set is about.
+fn memtable_reached_flush_threshold(
+    memtable: &MemTable,
+    max_memtable_size: usize,
+    incoming_batches: usize,
+) -> bool {
+    memtable.estimated_size() >= max_memtable_size
+        || memtable.batch_store().remaining_capacity() < incoming_batches
+}
+
+/// Flush a sealed replay memtable to a Lance generation, choosing the indexed
+/// path when secondary indexes are configured (mirroring the live memtable-flush
+/// handler). Commits the manifest, stamping `covered` as the generation's
+/// `replay_after_wal_entry_position` so a later reopen skips these entries.
+async fn flush_replayed_memtable(
+    flusher: &MemTableFlusher,
+    memtable: &MemTable,
+    epoch: u64,
+    covered: u64,
+    durable: usize,
+    index_configs: &[MemIndexConfig],
+) -> Result<()> {
+    if index_configs.is_empty() {
+        flusher.flush(memtable, epoch, covered, durable).await?;
+    } else {
+        Box::pin(flusher.flush_with_indexes(memtable, epoch, index_configs, covered, durable))
+            .await?;
+    }
+    Ok(())
 }
 
 /// Pair each primary-key column name with its field id (both derived from the
@@ -1001,6 +1113,10 @@ struct SharedWriterState {
     state: Arc<RwLock<WriterState>>,
     wal_flusher: Arc<WalFlusher>,
     wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
+    /// The index-apply task's channel. Separate from the WAL flusher's on
+    /// purpose: `TaskDispatcher::run` awaits `handle()` inline, so sharing one
+    /// would put a ~100ms S3 PUT in front of every latency-sensitive index apply.
+    index_apply_tx: mpsc::UnboundedSender<TriggerIndexApply>,
     memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
     config: ShardWriterConfig,
     schema: Arc<ArrowSchema>,
@@ -1019,6 +1135,7 @@ impl SharedWriterState {
         state: Arc<RwLock<WriterState>>,
         wal_flusher: Arc<WalFlusher>,
         wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
+        index_apply_tx: mpsc::UnboundedSender<TriggerIndexApply>,
         memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
         config: ShardWriterConfig,
         schema: Arc<ArrowSchema>,
@@ -1032,6 +1149,7 @@ impl SharedWriterState {
             state,
             wal_flusher,
             wal_flush_tx,
+            index_apply_tx,
             memtable_flush_tx,
             config,
             schema,
@@ -1043,59 +1161,100 @@ impl SharedWriterState {
         }
     }
 
+    /// Ask the index-apply task to cover `[indexed, end_batch_position)` of this
+    /// store. Cheap and idempotent: a range already covered is a no-op, which is
+    /// the common case under load, since one apply coalesces the puts queued
+    /// behind it.
+    fn trigger_index_apply(
+        &self,
+        batch_store: Arc<BatchStore>,
+        indexes: Arc<IndexStore>,
+        end_batch_position: usize,
+    ) -> Result<()> {
+        self.index_apply_tx
+            .send(TriggerIndexApply {
+                batch_store,
+                indexes,
+                end_batch_position,
+            })
+            .map_err(|_| Error::io("index apply channel closed"))
+    }
+
     /// Freeze the current memtable and send it to the flush handler.
     ///
     /// Takes `&mut WriterState` directly since caller already holds the lock.
     fn freeze_memtable(&self, state: &mut WriterState) -> Result<u64> {
-        let pending_wal_range = state.memtable.batch_store().pending_wal_flush_range();
+        let durable = self.wal_flusher.durable();
+        let pending_wal_range = state
+            .memtable
+            .batch_store()
+            .pending_wal_flush_range(durable);
         let last_wal_entry_position = state.last_flushed_wal_entry_position;
 
         let old_batch_store = state.memtable.batch_store();
-        let old_indexes = state.memtable.indexes_arc();
 
         let next_generation = state.memtable.generation() + 1;
-        let mut new_memtable = MemTable::with_capacity(
+        // The incoming memtable's batch 0 continues the writer's batch sequence
+        // where the outgoing one ends. Without this coordinate, local positions
+        // (which restart at 0 every rotation) cannot be mapped onto the
+        // writer-global durability cursor.
+        let next_global_offset = old_batch_store.global_end();
+        let mut new_memtable = MemTable::with_capacity_at(
             self.schema.clone(),
             next_generation,
             self.pk_field_ids.clone(),
             CacheConfig::default(),
             self.max_memtable_batches,
+            next_global_offset,
         )?;
 
-        // Build an IndexStore when there are user indexes *or* a primary key:
-        // the PK dedup index (and its flushed on-disk sidecar) is required for
-        // cross-generation dedup even when no secondary index is configured.
-        if !self.index_configs.is_empty() || !self.pk_columns.is_empty() {
-            let mut indexes = IndexStore::from_configs(
-                &self.index_configs,
-                self.max_memtable_rows,
-                self.max_memtable_batches,
-            )?;
+        // Always build and bind an IndexStore, even with no user indexes and no
+        // primary key. It is what carries the memtable's `indexed_count`, and
+        // binding it to the writer's cursors is what lets a reader derive the
+        // visible prefix — so an index-less memtable that skipped this would fall
+        // back to `visible == indexed` and publish rows before they were durable.
+        // (A PK memtable also needs the PK dedup index and its flushed sidecar.)
+        let mut indexes = IndexStore::from_configs(
+            &self.index_configs,
+            self.max_memtable_rows,
+            self.max_memtable_batches,
+        )?;
+        if !self.pk_columns.is_empty() {
             indexes.enable_pk_index(&pk_index_columns(&self.pk_columns, &self.pk_field_ids));
-            new_memtable.set_indexes_arc(Arc::new(indexes));
         }
+        indexes.set_durability(Arc::clone(self.wal_flusher.cursors()), next_global_offset);
+        new_memtable.set_indexes_arc(Arc::new(indexes));
 
         let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable);
         old_memtable.freeze(last_wal_entry_position);
+
+        // Set up completion tracking on the outgoing table before it is retained
+        // and before any fallible dispatch, so the retained table already carries
+        // its cells and a failed send below can poison-and-return without leaving
+        // partial state to unwind.
         let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion();
 
-        if pending_wal_range.is_some() {
+        // The outgoing memtable may still owe an index apply — the puts that
+        // filled it triggered one, but the task need not have drained yet, and
+        // this is the last chance to name that store. Its L0 flush is gated on
+        // the WAL append (below), not on indexing, so without this its tail could
+        // stay unindexed and invisible for the rest of its life.
+        let pending_index_apply = match old_memtable.indexes_arc() {
+            Some(old_indexes) if old_indexes.indexed_count() < old_batch_store.len() => {
+                Some((old_batch_store.clone(), old_indexes, old_batch_store.len()))
+            }
+            _ => None,
+        };
+
+        let pending_wal_flush = if pending_wal_range.is_some() {
             let completion_cell: WatchableOnceCell<
                 std::result::Result<WalFlushResult, WalFlushFailure>,
             > = WatchableOnceCell::new();
-            let completion_reader = completion_cell.reader();
-            old_memtable.set_wal_flush_completion(completion_reader);
-
-            let end_batch_position = old_batch_store.len();
-            self.wal_flusher.trigger_flush(
-                WalFlushSource::BatchStore {
-                    batch_store: old_batch_store,
-                    indexes: old_indexes,
-                },
-                end_batch_position,
-                Some(completion_cell),
-            )?;
-        }
+            old_memtable.set_wal_flush_completion(completion_cell.reader());
+            Some((old_batch_store.len(), completion_cell))
+        } else {
+            None
+        };
 
         let frozen_size = old_memtable.estimated_size();
         state.frozen_memtable_bytes += frozen_size;
@@ -1109,14 +1268,38 @@ impl SharedWriterState {
 
         let frozen_memtable = Arc::new(old_memtable);
 
-        // Keep this generation queryable past its manifest commit (swept after
-        // the grace by `SweepExpired`). Arc refcount, not a copy — the flush
-        // task holds it alive for the whole drain anyway.
+        // Retain the outgoing table in the read view *before* the fallible
+        // dispatches below. `state.memtable` was already replaced, so a failed
+        // send that returned here without this push would drop the table and its
+        // accepted rows would silently vanish from every scan. Keep it queryable
+        // past its manifest commit too (swept after the grace by `SweepExpired`);
+        // Arc refcount, not a copy — the flush task holds it alive anyway.
         state.frozen_memtables.push_back(FrozenMemTable {
             memtable: frozen_memtable.clone(),
             flushed_at_ms: None,
         });
 
+        // Dispatch can only fail if a background task's channel is already closed,
+        // i.e. the writer is being torn down. Poison so the read path fails fast
+        // with the typed error instead of serving the retained-but-never-durable
+        // tail, then return — the table stays in the read view.
+        if let Some((batch_store, indexes, end_batch_position)) = pending_index_apply {
+            self.trigger_index_apply(batch_store, indexes, end_batch_position)
+                .inspect_err(|e| self.wal_flusher.poison(e))?;
+        }
+
+        if let Some((end_batch_position, completion_cell)) = pending_wal_flush {
+            self.wal_flusher
+                .trigger_flush(
+                    WalFlushSource::BatchStore {
+                        batch_store: old_batch_store,
+                    },
+                    end_batch_position,
+                    Some(completion_cell),
+                )
+                .inspect_err(|e| self.wal_flusher.poison(e))?;
+        }
+
         debug!(
             "Frozen memtable generation {}, pending_count = {}",
             next_generation - 1,
@@ -1131,9 +1314,19 @@ impl SharedWriterState {
         Ok(next_generation)
     }
 
-    /// Track batch for WAL durability.
-    fn track_batch_for_wal(&self, batch_position: usize) -> super::wal::BatchDurableWatcher {
-        self.wal_flusher.track_batch(batch_position)
+    /// Watch for a write to become visible: indexed, and — in durable mode —
+    /// WAL-durable too.
+    ///
+    /// `target_indexed` is memtable-local; `target_durable` is writer-global.
+    /// Different coordinate spaces on purpose — see `WalFlusher::track_batch`.
+    fn track_batch_for_wal(
+        &self,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
+    ) -> super::wal::BatchDurableWatcher {
+        self.wal_flusher
+            .track_batch(indexes, target_indexed, target_durable)
     }
 
     /// Check if memtable flush is needed and trigger if so.
@@ -1144,8 +1337,10 @@ impl SharedWriterState {
             return Ok(());
         }
 
-        let should_flush = state.memtable.estimated_size() >= self.config.max_memtable_size
-            || state.memtable.is_batch_store_full();
+        // Checked post-insert: flush if there is no longer room for even one more
+        // batch (or the byte threshold is crossed). Same predicate replay uses.
+        let should_flush =
+            memtable_reached_flush_threshold(&state.memtable, self.config.max_memtable_size, 1);
 
         if should_flush {
             state.flush_requested = true;
@@ -1164,10 +1359,9 @@ impl SharedWriterState {
         let batch_count = state.memtable.batch_count();
         let total_bytes = state.memtable.estimated_size();
         let batch_store = state.memtable.batch_store();
-        let indexes = state.memtable.indexes_arc();
 
         // Check if there are any unflushed batches
-        let has_pending = batch_store.pending_wal_flush_count() > 0;
+        let has_pending = batch_store.pending_wal_flush_count(self.wal_flusher.durable()) > 0;
 
         // Check time-based trigger first
         let time_trigger = if let Some(interval) = self.config.max_wal_flush_interval {
@@ -1196,10 +1390,7 @@ impl SharedWriterState {
         // If time trigger fired, send a flush message
         if time_trigger.is_some() {
             let _ = self.wal_flush_tx.send(TriggerWalFlush {
-                source: WalFlushSource::BatchStore {
-                    batch_store,
-                    indexes,
-                },
+                source: WalFlushSource::BatchStore { batch_store },
                 end_batch_position: batch_count,
                 done: None,
             });
@@ -1224,7 +1415,6 @@ impl SharedWriterState {
             let _ = self.wal_flush_tx.send(TriggerWalFlush {
                 source: WalFlushSource::BatchStore {
                     batch_store: batch_store.clone(),
-                    indexes: indexes.clone(),
                 },
                 end_batch_position: batch_count,
                 done: None,
@@ -1334,6 +1524,21 @@ impl ShardWriter {
             ));
         }
 
+        // A durable writer needs a flush ticker to make progress, in either
+        // mode. With `durable_write` on, a put becomes durable only once its WAL
+        // append lands, and neither mode self-triggers that append per put — the
+        // background ticker drives it. Without an interval (or with a zero one,
+        // which tokio cannot schedule), a small put that never fills the
+        // size-triggered buffer would block until close. Reject the config here
+        // rather than let a put hang.
+        if config.durable_write && config.max_wal_flush_interval.is_none_or(|d| d.is_zero()) {
+            return Err(Error::invalid_input(
+                "durable_write requires a positive max_wal_flush_interval: with no \
+                 flush ticker a durable put has nothing to drive its WAL append and \
+                 would block until close",
+            ));
+        }
+
         // Callers pass the base schema; lance owns the `_tombstone` column and
         // appends it here so the memtable/generation schema = base + tombstone.
         // Idempotent, so a reopen that already extended the schema is a no-op.
@@ -1348,6 +1553,29 @@ impl ShardWriter {
             config.manifest_scan_batch_size,
         ));
 
+        // Derive PK metadata and run every side-effect-free validation *before*
+        // claiming the epoch. `claim_epoch` durably bumps the stored epoch and,
+        // for a successor, `write_fence_sentinel` fences the predecessor — so an
+        // open doomed by purely local input (an index config that disagrees with
+        // the schema) must fail here, before it can knock the healthy incumbent off
+        // the shard. Memtable-only: WAL-only mode has no indexes to validate.
+        let memtable_validation = if config.enable_memtable {
+            let lance_schema = Schema::try_from(schema.as_ref())?;
+            let pk_fields = lance_schema.unenforced_primary_key();
+            let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
+            let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
+
+            // Reject an index config that disagrees with the schema *before* a
+            // single row is accepted. Such a config fails deterministically on
+            // every insert, including inserts replayed from the WAL — so once a row
+            // is durable the shard can never reopen. Fail the open instead.
+            validate_index_configs(&index_configs, schema.as_ref(), &lance_schema, &pk_columns)?;
+
+            Some((pk_field_ids, pk_columns))
+        } else {
+            None
+        };
+
         // Claim the shard (epoch-based fencing) — done once, then shared
         // with the WalAppender via `with_claimed_epoch`.
         let (epoch, manifest) = manifest_store.claim_epoch(config.shard_spec_id).await?;
@@ -1389,7 +1617,12 @@ impl ShardWriter {
         }
 
         // Create WAL flusher backed by the shared appender.
-        let mut wal_flusher = WalFlusher::new(wal_appender);
+        // Build the cursors from *this writer's* config. `durable_write` is what
+        // decides whether durability is part of visibility, so a flusher that
+        // defaulted it would leave a non-durable put waiting on a durability
+        // cursor nothing ever advances.
+        let cursors = Arc::new(WriterCursors::new(config.durable_write));
+        let mut wal_flusher = WalFlusher::with_cursors(wal_appender, cursors);
 
         let (wal_flush_tx, wal_flush_rx) = mpsc::unbounded_channel();
         wal_flusher.set_flush_channel(wal_flush_tx.clone());
@@ -1399,11 +1632,15 @@ impl ShardWriter {
         let task_executor = Arc::new(TaskExecutor::new());
 
         let mode = if config.enable_memtable {
+            let (pk_field_ids, pk_columns) = memtable_validation
+                .expect("memtable_validation is Some when enable_memtable is true");
             Self::open_memtable_mode(
                 &config,
                 &schema,
                 &manifest,
                 &index_configs,
+                pk_field_ids,
+                pk_columns,
                 wal_flusher.clone(),
                 wal_flush_tx,
                 wal_flush_rx,
@@ -1445,6 +1682,8 @@ impl ShardWriter {
         schema: &Arc<ArrowSchema>,
         manifest: &ShardManifest,
         index_configs: &[MemIndexConfig],
+        pk_field_ids: Vec<i32>,
+        pk_columns: Vec<String>,
         wal_flusher: Arc<WalFlusher>,
         wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
         wal_flush_rx: mpsc::UnboundedReceiver<TriggerWalFlush>,
@@ -1457,50 +1696,93 @@ impl ShardWriter {
         stats: SharedWriteStats,
         task_executor: &Arc<TaskExecutor>,
     ) -> Result<WriterMode> {
-        // Create MemTable with primary key field IDs from schema
-        let lance_schema = Schema::try_from(schema.as_ref())?;
-        let pk_fields = lance_schema.unenforced_primary_key();
-        let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
-        let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
-        let mut memtable = MemTable::with_capacity(
-            schema.clone(),
-            manifest.current_generation,
-            pk_field_ids.clone(),
-            CacheConfig::default(),
-            config.max_memtable_batches,
-        )?;
-
-        // Create indexes if configured and set them on the MemTable. The
-        // PK-position index is enabled before any WAL replay below so replayed
-        // rows are recorded in it. A primary key alone (no secondary index)
-        // still needs the PK index so flush writes its on-disk dedup sidecar.
-        if !index_configs.is_empty() || !pk_columns.is_empty() {
+        // PK metadata and index/interval validation were resolved in `open`
+        // before the epoch was claimed (a doomed open must not fence the
+        // incumbent first).
+
+        // Build a fresh, cursor-bound memtable at a given generation and
+        // writer-global coordinate. Replay calls this for the first memtable and
+        // after every rotation. Always builds and binds an `IndexStore`, even
+        // with no user indexes and no primary key — see the note in
+        // `freeze_memtable` for why an index-less memtable still needs one.
+        let make_bound_memtable = |generation: u64, global_offset: usize| -> Result<MemTable> {
+            let mut memtable = MemTable::with_capacity_at(
+                schema.clone(),
+                generation,
+                pk_field_ids.clone(),
+                CacheConfig::default(),
+                config.max_memtable_batches,
+                global_offset,
+            )?;
             let mut indexes = IndexStore::from_configs(
                 index_configs,
                 config.max_memtable_rows,
                 config.max_memtable_batches,
             )?;
-            indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
+            if !pk_columns.is_empty() {
+                indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
+            }
+            indexes.set_durability(Arc::clone(wal_flusher.cursors()), global_offset);
             memtable.set_indexes_arc(Arc::new(indexes));
-        }
+            Ok(memtable)
+        };
+
+        // The flusher writes sealed memtables to Lance generations — both the
+        // ones replay seals below and the ones the live path freezes later.
+        let flusher = Arc::new(
+            MemTableFlusher::new(
+                object_store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                shard_id,
+                manifest_store.clone(),
+            )
+            .with_warmer(config.warmer.clone())
+            .with_storage_context(config.store_params.clone(), config.session.clone()),
+        );
 
         // Replay any WAL entries written after the last successfully-flushed
-        // generation. Each entry's writer_epoch is checked against ours; an
-        // entry with a strictly greater epoch indicates a successor writer
-        // claimed the shard between our `claim_epoch` and replay, so we
-        // abort the open with a fence error. The replay walked the tailer
-        // up to the WAL tip, so we hand the discovered next-write position
-        // straight to the appender — its first append skips the
-        // discover_next_position probe entirely.
-        let next_wal_position = replay_memtable_from_wal(
+        // SSTable, flushing sealed memtables to Lance SSTables as the batch
+        // store fills. Each entry's writer_epoch is checked against ours; an entry
+        // with a strictly greater epoch means a successor claimed the shard
+        // between our `claim_epoch` and replay, so we abort with a fence error.
+        // Replay walks the tailer to the WAL tip and returns the discovered
+        // next-write position, so the appender's first append skips the
+        // discover_next_position probe.
+        let ReplayResult {
+            active: memtable,
+            next_wal_position,
+        } = replay_memtable_from_wal(
             object_store.clone(),
             base_path.clone(),
             shard_id,
             epoch,
             manifest,
-            &mut memtable,
+            manifest.current_generation,
+            make_bound_memtable,
+            &flusher,
+            &wal_flusher,
+            index_configs,
+            config.max_memtable_size,
         )
         .await?;
+
+        // Mark the active memtable's replayed batches durable. They came *from*
+        // the WAL, and replay has already re-derived its indexes over them.
+        //
+        // Without this the durability cursor stays at the last sealed generation,
+        // so the next WAL flush re-covers the active tail: it re-appends the
+        // already-durable rows *and* re-inserts every replayed row into the
+        // indexes. None of the three in-memory indexes is idempotent (HNSW mints
+        // fresh node ids for the same row, FTS increments doc_count/df rather than
+        // recomputing them, BTree is a multiset), so a full scan keeps looking
+        // healthy while every index-accelerated query silently returns duplicates
+        // — and it compounds, because the WAL now holds those rows twice.
+        //
+        // `global_end()` is the writer-global batch count through this memtable,
+        // since its coordinate continues where the last sealed generation ended.
+        wal_flusher.advance_durable(memtable.batch_store().global_end());
+
         wal_flusher
             .wal_appender()
             .seed_next_position(next_wal_position)
@@ -1512,8 +1794,8 @@ impl ShardWriter {
         // it is durably reflected in this writer's memtable. We can't
         // seed from `manifest.wal_entry_position_last_seen` — that field
         // is bumped on every successful tailer read by other readers, so
-        // it may sit above what's actually covered by any flushed
-        // generation. Subtracting 1 from a fresh shard's `next_wal_position`
+        // it may sit above what's actually covered by any
+        // SSTable. Subtracting 1 from a fresh shard's `next_wal_position`
         // of `FIRST_WAL_ENTRY_POSITION` (= 1) yields 0, which correctly
         // means "no entry covered yet."
         let initial_covered_wal_entry_position = next_wal_position.saturating_sub(1);
@@ -1531,16 +1813,16 @@ impl ShardWriter {
 
         let (memtable_flush_tx, memtable_flush_rx) = mpsc::unbounded_channel();
 
-        let flusher = Arc::new(
-            MemTableFlusher::new(object_store, base_path, base_uri, shard_id, manifest_store)
-                .with_warmer(config.warmer.clone()),
-        );
-
         let backpressure = BackpressureController::new(config.clone());
 
         // Background WAL flush handler — parallel WAL I/O + index updates.
-        let wal_handler =
-            WalFlushHandler::new(wal_flusher.clone(), Some(state.clone()), stats.clone());
+        let wal_handler = WalFlushHandler::new(
+            wal_flusher.clone(),
+            Some(state.clone()),
+            None,
+            config.max_wal_flush_interval,
+            stats.clone(),
+        );
         task_executor.add_handler(
             "wal_flusher".to_string(),
             Box::new(wal_handler),
@@ -1548,13 +1830,14 @@ impl ShardWriter {
         )?;
 
         // Background MemTable flush handler — frozen memtable to Lance file.
-        // It rebuilds the same secondary indexes on each flushed generation.
+        // It rebuilds the same secondary indexes on each SSTable.
         let memtable_handler = MemTableFlushHandler::new(
             state.clone(),
             flusher,
+            wal_flusher.clone(),
             epoch,
             index_configs.to_vec(),
-            stats,
+            stats.clone(),
             config.frozen_memtable_grace,
         );
         task_executor.add_handler(
@@ -1563,11 +1846,27 @@ impl ShardWriter {
             memtable_flush_rx,
         )?;
 
+        // The index-apply task. Its own channel and its own dispatcher: the
+        // dispatcher awaits `handle()` inline, so sharing the WAL flusher's
+        // channel would queue every index apply behind a ~100ms S3 PUT.
+        let (index_apply_tx, index_apply_rx) = mpsc::unbounded_channel();
+        let index_handler = IndexApplyHandler {
+            cursors: Arc::clone(wal_flusher.cursors()),
+            wal_flusher: wal_flusher.clone(),
+            stats,
+        };
+        task_executor.add_handler(
+            "index_applier".to_string(),
+            Box::new(index_handler),
+            index_apply_rx,
+        )?;
+
         // Shared state used by `put()` to dispatch trigger checks.
         let writer_state = Arc::new(SharedWriterState::new(
             state.clone(),
             wal_flusher,
             wal_flush_tx,
+            index_apply_tx,
             memtable_flush_tx,
             config.clone(),
             schema.clone(),
@@ -1593,9 +1892,25 @@ impl ShardWriter {
         stats: SharedWriteStats,
         task_executor: &Arc<TaskExecutor>,
     ) -> Result<WriterMode> {
+        // The pending queue is shared with the flush handler so a background
+        // tick can resolve to the batches still owed an append — the WAL-only
+        // analog of resolving a tick against the durability cursor in MemTable
+        // mode. A durable put waits on the durability cursor the handler's
+        // append advances, so the ticker must run (`open` rejects durable +
+        // no-interval); a non-durable writer may pass no interval and rely on
+        // the size/close triggers alone.
+        let state = Arc::new(WalOnlyState::default());
+
         // Background WAL flush handler — no MemTable state to consult, so
-        // pass `None` for the frozen-vs-active detection.
-        let wal_handler = WalFlushHandler::new(wal_flusher, None, stats);
+        // pass `None` for the frozen-vs-active detection; the pending queue and
+        // the flush interval drive the background append instead.
+        let wal_handler = WalFlushHandler::new(
+            wal_flusher,
+            None,
+            Some(state.clone()),
+            config.max_wal_flush_interval,
+            stats,
+        );
         task_executor.add_handler(
             "wal_flusher".to_string(),
             Box::new(wal_handler),
@@ -1610,7 +1925,7 @@ impl ShardWriter {
         let backpressure = BackpressureController::new(config.clone());
 
         Ok(WriterMode::WalOnly {
-            state: Arc::new(WalOnlyState::default()),
+            state,
             wal_flush_tx,
             trigger: StdRwLock::new(WalOnlyTriggerState::default()),
             backpressure,
@@ -1850,46 +2165,74 @@ impl ShardWriter {
             // 1. Insert all batches into memtable atomically
             let results = state.memtable.insert_batches_only(batches).await?;
 
-            // Get batch position range
+            // 2. Capture the store the batches actually landed in, *before* step
+            //    4 below can freeze and swap the active memtable. Reading it
+            //    afterwards hands the flush trigger the **new** store paired with
+            //    the **old** store's end position, so the new store's watermark
+            //    jumps past batches that were never appended.
+            let batch_store = state.memtable.batch_store();
+            let indexes = state.memtable.indexes_arc();
+
             let start_pos = results.first().map(|(pos, _, _)| *pos).unwrap_or(0);
             let end_pos = results.last().map(|(pos, _, _)| pos + 1).unwrap_or(0);
             let batch_positions = start_pos..end_pos;
 
-            // 2. Track last batch for WAL durability
-            let durable_watcher = writer_state.track_batch_for_wal(end_pos.saturating_sub(1));
+            // 3. Watch for this write to become *visible*: indexed, and — under
+            //    `durable_write` — WAL-durable too.
+            //
+            //    The two targets live in different coordinate spaces. `end_pos`
+            //    is memtable-local, which is what the index apply works in. The
+            //    durability cursor is writer-global, because batch positions
+            //    restart at 0 in every memtable while that cursor spans the
+            //    writer's whole life — a local durable target would already be
+            //    satisfied by a *previous* memtable's appends, so the first N
+            //    puts into every post-rotation memtable would ack as durable with
+            //    no WAL append ever happening.
+            let durable_watcher = writer_state.track_batch_for_wal(
+                indexes.clone(),
+                end_pos,
+                batch_store.global_offset() + end_pos,
+            );
 
-            // 3. Check if WAL flush should be triggered
+            // 4. Check if WAL flush should be triggered
             writer_state.maybe_trigger_wal_flush(&mut state);
 
-            // 4. Check if memtable flush is needed
+            // 5. Check if memtable flush is needed (may freeze and rotate)
             if let Err(e) = writer_state.maybe_trigger_memtable_flush(&mut state) {
                 warn!("Failed to trigger memtable flush: {}", e);
             }
 
-            // Get batch_store and indexes while we have the lock (for durable_write case)
-            let batch_store = state.memtable.batch_store();
-            let indexes = state.memtable.indexes_arc();
-
             (batch_positions, durable_watcher, batch_store, indexes)
         }; // Lock released here
 
         self.stats.record_put(start.elapsed());
 
-        // Trigger the flush here (outside the lock) so the watcher can resolve;
-        // only the `wait()` is the caller's to schedule.
-        let watcher = if self.config.durable_write {
-            self.wal_flusher.trigger_flush(
-                WalFlushSource::BatchStore {
-                    batch_store,
-                    indexes,
-                },
-                batch_positions.end,
-                None,
-            )?;
-            Some(durable_watcher)
-        } else {
-            None
-        };
+        // Trigger the index apply, in **both** modes. This is what makes reads
+        // read-your-writes regardless of `durable_write`: skipping durability now
+        // costs the caller durability only, not visibility. It is cheap
+        // (in-memory, ~ms), so there is no reason to batch it onto the WAL's
+        // schedule — that schedule exists to bound S3 API cost, which an
+        // in-memory index apply does not incur.
+        if let Some(indexes) = indexes {
+            writer_state.trigger_index_apply(batch_store, indexes, batch_positions.end)?;
+        }
+
+        // The WAL append is *not* triggered here. It happens on the background
+        // ticker (and on the size trigger, and at freeze/close), which is the only
+        // way the flush interval can mean anything: while every durable put
+        // triggered its own append, the interval could add a redundant trigger but
+        // never delay or batch one.
+        //
+        // The cost is real and accepted: a single client's sequential *durable*
+        // throughput drops from ~10 writes/sec (one PUT round-trip) to roughly one
+        // per tick. That is a policy choice — the interval should mean what it
+        // says, and S3 API cost should be bounded. Latency-sensitive callers want
+        // `durable_write: false`, which now costs them durability only, not
+        // visibility.
+
+        // The watcher is returned in both modes now. A non-durable put still
+        // waits — for its index apply (~ms), not for an S3 PUT (~100ms).
+        let watcher = Some(durable_watcher);
 
         Ok((WriteResult { batch_positions }, watcher))
     }
@@ -1920,54 +2263,48 @@ impl ShardWriter {
         // Push batches into the pending queue and capture the assigned
         // [start, end) range. `next_batch_position` is monotonic across the
         // writer's lifetime; positions are not BatchStore indices but they
-        // are used the same way for durability tracking.
+        // are used the same way for durability tracking. Because the queue is
+        // strict FIFO with contiguous positions and its front always sits at
+        // the durability cursor, `end` is exactly the writer-global durable
+        // count this put reaches once its append lands — no globalizing offset
+        // as in MemTable mode, which restarts positions per generation.
         let batch_positions = state.push(batches);
 
-        // Time- and size-based triggers, mirroring MemTable mode but reading
-        // pending bytes from `WalOnlyState` instead of an active MemTable.
-        // Only fires for non-durable writes; durable writes go through the
-        // explicit done-cell path below so flush errors (e.g., fence) reach
-        // the caller.
-        if !self.config.durable_write {
-            let target_position = batch_positions.end;
-            let pending_bytes = state.estimated_size();
-            self.maybe_trigger_wal_flush_wal_only(
-                state,
-                wal_flush_tx,
-                trigger,
-                target_position,
-                pending_bytes,
-            );
-        }
+        // Under `durable_write` the put becomes durable only once its WAL
+        // append lands. Track it on the writer-global durability cursor *before*
+        // triggering, so a flush that completes between here and the wait is not
+        // missed: the watcher recomputes visibility from the cursor rather than
+        // latching a one-shot wake. WAL-only mode has no indexes, so the
+        // index-visibility half of the watcher is a no-op (`None`, target 0).
+        let durable_watcher = self
+            .config
+            .durable_write
+            .then(|| self.wal_flusher.track_batch(None, 0, batch_positions.end));
+
+        // Time- and size-based triggers on the write path, for durable and
+        // non-durable puts alike — mirroring MemTable mode's
+        // `maybe_trigger_wal_flush`. The background ticker drives the append
+        // too; whichever fires first wins, and a redundant trigger is a cheap
+        // no-op because the flush snapshot/commit is idempotent.
+        self.maybe_trigger_wal_flush_wal_only(
+            state,
+            wal_flush_tx,
+            trigger,
+            batch_positions.end,
+            state.estimated_size(),
+        );
 
         self.stats.record_put(start.elapsed());
 
-        // For durable writes, trigger an immediate flush and wait for the
-        // done cell. Using the done cell instead of the durability watermark
-        // watcher ensures flush errors (e.g., the WalAppender returning a
-        // fence error) propagate back to `put` instead of hanging.
-        if self.config.durable_write {
-            let done = WatchableOnceCell::new();
-            let reader = done.reader();
-            self.wal_flusher.trigger_flush(
-                WalFlushSource::WalOnly {
-                    state: state.clone(),
-                },
-                batch_positions.end,
-                Some(done),
-            )?;
-            let mut reader = reader;
-            match reader.await_value().await {
-                Some(Ok(_)) => {}
-                // Rebuild the typed error (peer fence vs. persistence-failure
-                // self-fence) so a WAL-only durable caller can tell them apart.
-                Some(Err(failure)) => return Err(failure.into_error()),
-                None => {
-                    return Err(Error::io(
-                        "WAL flush handler exited before reporting durability",
-                    ));
-                }
-            }
+        // Durable writes wait on the durability cursor, advanced by the append
+        // the ticker (or the trigger above) drives — the same watermark path
+        // MemTable mode uses. A terminal flush failure (a peer fence or an
+        // exhausted-retry persistence self-fence) poisons the writer and wakes
+        // the waiter with that typed error instead of leaving it to hang. This
+        // is why `open` rejects `durable_write` with no flush ticker: nothing
+        // else would advance the cursor a small put is parked on.
+        if let Some(mut watcher) = durable_watcher {
+            watcher.wait().await?;
         }
 
         Ok(WriteResult { batch_positions })
@@ -2067,18 +2404,24 @@ impl ShardWriter {
 
     /// Get current MemTable statistics. Returns an error in WAL-only mode
     /// (no MemTable exists).
+    ///
+    /// Deliberately does *not* `check_poisoned`, unlike the read and write
+    /// paths: a poisoned writer is exactly when an operator most needs to see
+    /// its state, and the caller deciding whether to evict reads these stats.
     pub async fn memtable_stats(&self) -> Result<MemTableStats> {
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         let batch_store = state.memtable.batch_store();
-        let pending_wal = batch_store.pending_wal_flush_stats();
+        let durable = self.wal_flusher.durable();
+        let pending_wal = batch_store.pending_wal_flush_stats(durable);
         Ok(MemTableStats {
             row_count: state.memtable.row_count(),
             batch_count: state.memtable.batch_count(),
             estimated_size: state.memtable.estimated_size(),
             generation: state.memtable.generation(),
             max_buffered_batch_position: batch_store.max_buffered_batch_position(),
-            max_flushed_batch_position: batch_store.max_flushed_batch_position(),
+            durable_batch_count: durable,
+            global_offset: batch_store.global_offset(),
             pending_wal_start_batch_position: pending_wal.start_batch_position,
             pending_wal_end_batch_position: pending_wal.end_batch_position,
             pending_wal_batch_count: pending_wal.batch_count,
@@ -2092,11 +2435,12 @@ impl ShardWriter {
     /// The scanner provides read access to all data currently in the MemTable,
     /// with optional filtering, projection, and index support.
     ///
-    /// The scanner captures the current `max_visible_batch_position` from the
+    /// The scanner captures the current `visible_count` from the
     /// `IndexStore` at construction time to ensure consistent visibility.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn scan(&self) -> Result<MemTableScanner> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(state.memtable.scan())
@@ -2106,10 +2450,11 @@ impl ShardWriter {
     /// Prefer [`Self::in_memory_memtable_refs`] on the read path — it also
     /// carries frozen-awaiting-flush generations.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn active_memtable_ref(
         &self,
     ) -> Result<crate::dataset::mem_wal::scanner::InMemoryMemTableRef> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(in_memory_ref(&state.memtable))
@@ -2121,10 +2466,11 @@ impl ShardWriter {
     /// path uses this instead of [`Self::active_memtable_ref`] so a
     /// concurrent reader sees no hole while a flush drains.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn in_memory_memtable_refs(
         &self,
     ) -> Result<crate::dataset::mem_wal::scanner::InMemoryMemTables> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(crate::dataset::mem_wal::scanner::InMemoryMemTables {
@@ -2268,12 +2614,65 @@ impl ShardWriter {
         Ok(())
     }
 
+    /// Send the close-time final WAL flush and await its completion.
+    ///
+    /// Sends directly on the flush channel rather than via
+    /// [`WalFlusher::trigger_flush`]: the latter silently returns `Ok` when the
+    /// flusher's `flush_tx` is unset, which would let close report success
+    /// without ever persisting the final WAL entry. A closed send channel must
+    /// surface as an error here so close never acknowledges durability it did
+    /// not achieve.
+    async fn flush_final_wal(
+        wal_flush_tx: &mpsc::UnboundedSender<TriggerWalFlush>,
+        source: WalFlushSource,
+        end_batch_position: usize,
+    ) -> Result<()> {
+        let done = WatchableOnceCell::new();
+        let mut reader = done.reader();
+        if wal_flush_tx
+            .send(TriggerWalFlush {
+                source,
+                end_batch_position,
+                done: Some(done),
+            })
+            .is_err()
+        {
+            return Err(Error::io("WAL flush channel closed during close"));
+        }
+
+        match reader.await_value().await {
+            Some(Ok(_)) => Ok(()),
+            Some(Err(failure)) => Err(failure.into_error()),
+            None => Err(Error::io(
+                "WAL flush handler exited before reporting durability during close",
+            )),
+        }
+    }
+
+    fn merge_close_stage(
+        close_result: Result<()>,
+        stage: &str,
+        stage_result: Result<()>,
+    ) -> Result<()> {
+        if let (Err(_), Err(stage_error)) = (&close_result, &stage_result) {
+            warn!("Close stage '{stage}' also failed: {stage_error}");
+        }
+        close_result.and(stage_result)
+    }
+
     /// Close the writer gracefully.
     ///
     /// Flushes pending data and shuts down background tasks.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if pending WAL data cannot be persisted, an active or
+    /// frozen MemTable cannot be flushed, a flush handler exits before reporting
+    /// completion, or background tasks cannot be shut down.
     #[instrument(name = "sw_close", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))]
     pub async fn close(self) -> Result<()> {
         info!("Closing ShardWriter for shard {}", self.config.shard_id);
+        let mut close_result: Result<()> = Ok(());
 
         match &self.mode {
             WriterMode::MemTable {
@@ -2281,31 +2680,46 @@ impl ShardWriter {
                 writer_state,
                 ..
             } => {
-                // Send final WAL flush message and wait for completion
+                // Drain *both* tasks against the active memtable. The index apply
+                // and the WAL append are independent now, so closing has to
+                // settle both: the L0 flush below turns this memtable into a
+                // Lance generation, and a generation whose indexes never saw the
+                // tail is a generation with a hole in it.
                 let st = state.read().await;
                 let batch_store = st.memtable.batch_store();
                 let indexes = st.memtable.indexes_arc();
                 let batch_count = st.memtable.batch_count();
                 drop(st);
 
+                if batch_count > 0
+                    && let Some(indexes) = indexes
+                    && indexes.indexed_count() < batch_count
+                {
+                    let mut watcher = self.wal_flusher.track_batch(
+                        Some(Arc::clone(&indexes)),
+                        batch_count,
+                        0, // durability is settled by the WAL flush below
+                    );
+                    writer_state.trigger_index_apply(
+                        Arc::clone(&batch_store),
+                        indexes,
+                        batch_count,
+                    )?;
+                    watcher.wait().await?;
+                }
+
                 if batch_count > 0 {
-                    let done = WatchableOnceCell::new();
-                    let reader = done.reader();
-                    if writer_state
-                        .wal_flush_tx
-                        .send(TriggerWalFlush {
-                            source: WalFlushSource::BatchStore {
-                                batch_store,
-                                indexes,
-                            },
-                            end_batch_position: batch_count,
-                            done: Some(done),
-                        })
-                        .is_ok()
-                    {
-                        let mut reader = reader;
-                        let _ = reader.await_value().await;
-                    }
+                    // Append-only source: on this branch the index apply is a
+                    // separate task (drained above), so the final WAL flush carries
+                    // no indexes. #7769's failure propagation still applies.
+                    let stage_result = Self::flush_final_wal(
+                        &writer_state.wal_flush_tx,
+                        WalFlushSource::BatchStore { batch_store },
+                        batch_count,
+                    )
+                    .await;
+                    close_result =
+                        Self::merge_close_stage(close_result, "final WAL flush", stage_result);
                 }
 
                 // Freeze the active memtable (if any rows) so it joins the
@@ -2321,22 +2735,37 @@ impl ShardWriter {
                 // Propagate any freeze error: at close time the caller
                 // has explicitly asked for full durability, so silently
                 // dropping a freeze failure would lose data without any
-                // signal. If freeze fails, surface the error rather than
-                // continuing on to drain only the pre-existing frozen
-                // memtables (whose flushes can still be waited on, but
-                // the caller now knows the close was incomplete).
+                // signal. If freeze fails, its error is recorded as the
+                // first causal failure, but close still drains any
+                // pre-existing frozen MemTable watchers so a successor
+                // failure is logged without replacing the first error.
                 let watchers: Vec<_> = {
                     let mut st = state.write().await;
                     if st.memtable.row_count() > 0 {
-                        writer_state.freeze_memtable(&mut st)?;
+                        let freeze_result = writer_state.freeze_memtable(&mut st).map(|_| ());
+                        close_result = Self::merge_close_stage(
+                            close_result,
+                            "active MemTable freeze",
+                            freeze_result,
+                        );
                     }
                     st.frozen_flush_watchers
                         .iter()
                         .map(|(_, w)| w.clone())
                         .collect()
                 };
-                for mut w in watchers {
-                    let _ = w.await_value().await;
+                for mut watcher in watchers {
+                    let stage_result = match watcher.await_value().await {
+                        Some(durability) => durability.into_result(),
+                        None => Err(Error::io(
+                            "MemTable flush handler exited before reporting completion during close",
+                        )),
+                    };
+                    close_result = Self::merge_close_stage(
+                        close_result,
+                        "frozen MemTable flush watcher",
+                        stage_result,
+                    );
                 }
             }
             WriterMode::WalOnly {
@@ -2349,30 +2778,32 @@ impl ShardWriter {
                 let pending = state.batch_count();
                 let end_position = state.next_batch_position();
                 if pending > 0 {
-                    let done = WatchableOnceCell::new();
-                    let reader = done.reader();
-                    if wal_flush_tx
-                        .send(TriggerWalFlush {
-                            source: WalFlushSource::WalOnly {
-                                state: state.clone(),
-                            },
-                            end_batch_position: end_position,
-                            done: Some(done),
-                        })
-                        .is_ok()
-                    {
-                        let mut reader = reader;
-                        let _ = reader.await_value().await;
-                    }
+                    let stage_result = Self::flush_final_wal(
+                        wal_flush_tx,
+                        WalFlushSource::WalOnly {
+                            state: state.clone(),
+                        },
+                        end_position,
+                    )
+                    .await;
+                    close_result =
+                        Self::merge_close_stage(close_result, "final WAL flush", stage_result);
                 }
             }
         }
 
         // Shutdown background tasks
-        self.task_executor.shutdown_all().await?;
-
-        info!("ShardWriter closed for shard {}", self.config.shard_id);
-        Ok(())
+        let shutdown_result = self.task_executor.shutdown_all().await;
+        let close_result = Self::merge_close_stage(close_result, "task shutdown", shutdown_result);
+
+        match &close_result {
+            Ok(()) => info!("ShardWriter closed for shard {}", self.config.shard_id),
+            Err(error) => warn!(
+                "ShardWriter close for shard {} failed: {error}",
+                self.config.shard_id
+            ),
+        }
+        close_result
     }
 }
 
@@ -2384,7 +2815,12 @@ pub struct MemTableStats {
     pub estimated_size: usize,
     pub generation: u64,
     pub max_buffered_batch_position: Option<usize>,
-    pub max_flushed_batch_position: Option<usize>,
+    /// Writer-global count of WAL-durable batches. Exclusive: 0 means none.
+    /// Compare against `global_offset + batch_count` to see what this memtable
+    /// still owes the WAL.
+    pub durable_batch_count: usize,
+    /// Writer-global coordinate of this memtable's batch 0.
+    pub global_offset: usize,
     pub pending_wal_start_batch_position: Option<usize>,
     pub pending_wal_end_batch_position: Option<usize>,
     pub pending_wal_batch_count: usize,
@@ -2399,16 +2835,80 @@ pub struct WalStats {
     pub next_wal_entry_position: u64,
 }
 
-/// Background handler for WAL flush operations.
+/// The oldest store that still owes the WAL an append, or `None` when everything
+/// is durable.
+///
+/// Ordering is the whole point. WAL entry positions are assigned in append-call
+/// order; replay walks them ascending; row positions follow; and primary-key
+/// recency is "newest visible row position wins". So appending a newer memtable
+/// ahead of an older one's tail silently inverts dedup after a crash.
+///
+/// Taking "the active memtable" would do exactly that, because a timer tick
+/// enqueued before a freeze is handled after it and would resolve to the incoming
+/// memtable. Selecting by cursor instead makes the target a function of what is
+/// actually durable, not of when the timer happened to fire.
+fn next_pending_store(
+    frozen: impl Iterator<Item = Arc<BatchStore>>,
+    active: Arc<BatchStore>,
+    durable: usize,
+) -> Option<Arc<BatchStore>> {
+    frozen
+        .chain(std::iter::once(active))
+        .find(|store| store.global_end() > durable)
+}
+
+/// The index-apply task: one sequential consumer of the index-apply channel.
 ///
-/// This handler does parallel WAL I/O + index updates during flush.
-/// Indexes are passed through the TriggerWalFlush message.
+/// Sequential consumption is the safety property. `HnswGraph::insert_batch` hard-
+/// rejects any range whose start is not its `indexed_len`, so the apply must see
+/// contiguous, in-order ranges — and a single consumer guarantees that
+/// regardless of how many putters race behind it. Ordering comes from the task,
+/// not from a flush interval, which is why triggering per-put is exactly as safe
+/// as triggering on a timer, and lets a put become visible in milliseconds
+/// instead of waiting on an S3 round-trip.
+struct IndexApplyHandler {
+    cursors: Arc<WriterCursors>,
+    wal_flusher: Arc<WalFlusher>,
+    stats: SharedWriteStats,
+}
+
+#[async_trait]
+impl MessageHandler<TriggerIndexApply> for IndexApplyHandler {
+    async fn handle(&mut self, message: TriggerIndexApply) -> Result<()> {
+        match apply_index_range(&self.cursors, message).await {
+            Ok(applied) => {
+                // A coalesced no-op indexes nothing (`rows_indexed == 0`);
+                // recording it would inflate the count and skew avg latency.
+                if applied.rows_indexed > 0 {
+                    self.stats
+                        .record_index_update(applied.duration, applied.rows_indexed);
+                }
+                Ok(())
+            }
+            // An index apply cannot be partially rolled back, so a failure is
+            // terminal: poison, and let reopen rebuild the indexes from the WAL.
+            // See the note in `WalFlusher::flush_from_batch_store`.
+            Err(e) => {
+                self.wal_flusher.poison(&e);
+                Err(e)
+            }
+        }
+    }
+}
+
 struct WalFlushHandler {
     wal_flusher: Arc<WalFlusher>,
     /// MemTable-mode writer state, used to detect "frozen vs active" flushes
     /// via Arc::ptr_eq on the active batch_store. `None` when running in
     /// WAL-only mode (no MemTable, no frozen-vs-active distinction).
     memtable_state: Option<Arc<RwLock<WriterState>>>,
+    /// WAL-only-mode pending queue, so a background tick can resolve to the
+    /// batches still owed an append. `None` in MemTable mode. Exactly one of
+    /// `memtable_state` / `wal_only_state` is `Some`.
+    wal_only_state: Option<Arc<WalOnlyState>>,
+    /// How often to append in the background. `None` disables the ticker, leaving
+    /// the append size-triggered (and freeze/close-triggered) only.
+    flush_interval: Option<Duration>,
     stats: SharedWriteStats,
 }
 
@@ -2416,11 +2916,15 @@ impl WalFlushHandler {
     fn new(
         wal_flusher: Arc<WalFlusher>,
         memtable_state: Option<Arc<RwLock<WriterState>>>,
+        wal_only_state: Option<Arc<WalOnlyState>>,
+        flush_interval: Option<Duration>,
         stats: SharedWriteStats,
     ) -> Self {
         Self {
             wal_flusher,
             memtable_state,
+            wal_only_state,
+            flush_interval,
             stats,
         }
     }
@@ -2428,6 +2932,33 @@ impl WalFlushHandler {
 
 #[async_trait]
 impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
+    /// Append periodically in the background.
+    ///
+    /// This is what the flush interval was always supposed to mean. It routed to
+    /// a timer that was only ever *evaluated on the write path*, so it could add
+    /// a redundant trigger but never delay or batch one — with every durable put
+    /// triggering its own append, tuning the knob did nothing at all.
+    ///
+    /// The ticker exists to bound S3 API cost: an append is a PUT, billed per
+    /// call, and it is the only thing on this schedule. The index apply is not —
+    /// it is in-memory and free to batch, so it runs per-put on its own task.
+    fn tickers(&mut self) -> Vec<(Duration, MessageFactory<TriggerWalFlush>)> {
+        // No interval => no ticker. A zero interval would panic in tokio.
+        let Some(interval) = self.flush_interval.filter(|d| !d.is_zero()) else {
+            return vec![];
+        };
+        // The tick names no store: `MessageFactory` is synchronous and cannot take
+        // the async state lock. `handle()` resolves it against the cursor.
+        vec![(
+            interval,
+            Box::new(|| TriggerWalFlush {
+                source: WalFlushSource::NextPending,
+                end_batch_position: 0,
+                done: None,
+            }),
+        )]
+    }
+
     async fn handle(&mut self, message: TriggerWalFlush) -> Result<()> {
         let TriggerWalFlush {
             source,
@@ -2435,6 +2966,16 @@ impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
             done,
         } = message;
 
+        // A timer tick names no store — resolve it now, at handle time.
+        let (source, end_batch_position) = match source {
+            WalFlushSource::NextPending => match self.resolve_next_pending().await {
+                Some(resolved) => resolved,
+                // Everything is already durable; the tick has nothing to do.
+                None => return Ok(()),
+            },
+            other => (other, end_batch_position),
+        };
+
         let result = self.do_flush(source, end_batch_position).await;
 
         // Propagate the just-appended WAL entry position back into the
@@ -2465,6 +3006,64 @@ impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
 }
 
 impl WalFlushHandler {
+    /// Pick the store the WAL still owes an append, oldest first.
+    ///
+    /// **WAL entries must be appended in global batch-position order for the
+    /// writer's lifetime.** `WalAppender::append` assigns each entry's position
+    /// from its own counter, in call order; replay walks those positions
+    /// ascending and assigns row positions in that order; and primary-key recency
+    /// is "newest visible row position wins". So append order fixes dedup order.
+    /// Append two memtables out of order and replay silently hands the dedup to
+    /// the *stale* row — corruption that survives the crash that caused it, and
+    /// that a full scan cannot see.
+    ///
+    /// Resolving to "the active memtable" would break exactly that: a tick
+    /// enqueued before a freeze is handled after it, resolves to the incoming
+    /// memtable, and appends its batches ahead of the outgoing memtable's tail.
+    /// So the target is a function of `durable`, not of when the timer fired.
+    ///
+    /// Safe to walk the frozen list because a store that still owes an append
+    /// cannot be swept: its L0 flush is blocked on the completion cell that only
+    /// that append fires.
+    ///
+    /// In WAL-only mode there is a single FIFO pending queue and no memtable
+    /// rotation, so the ordering hazard above cannot arise: the tick resolves to
+    /// the queue whenever it holds un-appended batches.
+    async fn resolve_next_pending(&self) -> Option<(WalFlushSource, usize)> {
+        if let Some(state_lock) = self.memtable_state.as_ref() {
+            let state = state_lock.read().await;
+            let durable = self.wal_flusher.durable();
+
+            return next_pending_store(
+                state
+                    .frozen_memtables
+                    .iter()
+                    .map(|frozen| frozen.memtable.batch_store()),
+                state.memtable.batch_store(),
+                durable,
+            )
+            .map(|store| {
+                let end = store.len();
+                (WalFlushSource::BatchStore { batch_store: store }, end)
+            });
+        }
+
+        let state = self.wal_only_state.as_ref()?;
+        if state.batch_count() == 0 {
+            // Everything already appended; the tick has nothing to do.
+            return None;
+        }
+        // `flush_from_wal_only` snapshots the whole queue, so the end position
+        // is informational here; carry the next position for symmetry.
+        let end = state.next_batch_position();
+        Some((
+            WalFlushSource::WalOnly {
+                state: Arc::clone(state),
+            },
+            end,
+        ))
+    }
+
     /// Unified flush method for both active and frozen memtables and for
     /// WAL-only mode.
     ///
@@ -2484,18 +3083,6 @@ impl WalFlushHandler {
     ) -> Result<WalFlushResult> {
         let start = Instant::now();
 
-        // Whether this flush actually updates any in-memory indexes — only
-        // a BatchStore source carrying a non-empty `IndexStore` does. Used
-        // to gate the `record_index_update` stat so WAL-only flushes don't
-        // pollute the index-update counters.
-        let has_indexes = matches!(
-            &source,
-            WalFlushSource::BatchStore {
-                indexes: Some(_),
-                ..
-            }
-        );
-
         // Early-out for BatchStore sources where the watermark already
         // covers the requested end position. Detection of "frozen flush"
         // requires the active memtable's batch_store; WAL-only handlers
@@ -2503,8 +3090,7 @@ impl WalFlushHandler {
         // BatchStore source, so the early-out simplifies to the watermark
         // comparison.
         if let WalFlushSource::BatchStore { batch_store, .. } = &source {
-            let max_flushed = batch_store.max_flushed_batch_position();
-            let flushed_up_to = max_flushed.map(|p| p + 1).unwrap_or(0);
+            let flushed_up_to = batch_store.local_end(self.wal_flusher.durable());
             let is_frozen_flush = if let Some(state_lock) = &self.memtable_state {
                 let state = state_lock.read().await;
                 !Arc::ptr_eq(batch_store, &state.memtable.batch_store())
@@ -2529,12 +3115,6 @@ impl WalFlushHandler {
             self.stats
                 .record_wal_flush(start.elapsed(), flush_result.wal_bytes);
             self.stats.record_wal_io(flush_result.wal_io_duration);
-            if has_indexes {
-                self.stats.record_index_update(
-                    flush_result.index_update_duration,
-                    flush_result.rows_indexed,
-                );
-            }
         }
 
         Ok(flush_result)
@@ -2550,10 +3130,13 @@ impl WalFlushHandler {
 struct MemTableFlushHandler {
     state: Arc<RwLock<WriterState>>,
     flusher: Arc<MemTableFlusher>,
+    /// Source of the writer-global durability cursor, which the L0 flush asserts
+    /// covers the whole frozen memtable before it writes a generation.
+    wal_flusher: Arc<WalFlusher>,
     epoch: u64,
-    /// Secondary index configs to rebuild on each flushed generation. When
+    /// Secondary index configs to rebuild on each SSTable. When
     /// non-empty the handler flushes via [`MemTableFlusher::flush_with_indexes`]
-    /// so queries over flushed generations use index lookups instead of full
+    /// so queries over SSTables use index lookups instead of full
     /// scans — and so vector search's index-only `fast_search` can see the data
     /// at all.
     index_configs: Vec<MemIndexConfig>,
@@ -2564,9 +3147,11 @@ struct MemTableFlushHandler {
 }
 
 impl MemTableFlushHandler {
+    #[allow(clippy::too_many_arguments)]
     fn new(
         state: Arc<RwLock<WriterState>>,
         flusher: Arc<MemTableFlusher>,
+        wal_flusher: Arc<WalFlusher>,
         epoch: u64,
         index_configs: Vec<MemIndexConfig>,
         stats: SharedWriteStats,
@@ -2575,6 +3160,7 @@ impl MemTableFlushHandler {
         Self {
             state,
             flusher,
+            wal_flusher,
             epoch,
             index_configs,
             stats,
@@ -2681,14 +3267,20 @@ impl MemTableFlushHandler {
             let covered_wal_entry_position = wal_flushed_position
                 .or_else(|| memtable.frozen_at_wal_entry_position())
                 .unwrap_or(0);
-            // Rebuild secondary indexes on the flushed generation so later
+            // Rebuild secondary indexes on the SSTable so later
             // queries hit an index instead of scanning. Skip the extra
             // dataset open when there are no indexes to build. The indexed
             // path's future is boxed to keep this async block's nesting
             // under the type-layout recursion limit.
+            // Read the durability cursor *after* the WAL-append completion above,
+            // not before: the append that makes this memtable durable is the very
+            // thing we just waited on, so a cursor sampled earlier would still be
+            // short of it and trip the flush precondition.
+            let durable = self.wal_flusher.durable();
+
             if self.index_configs.is_empty() {
                 self.flusher
-                    .flush(&memtable, self.epoch, covered_wal_entry_position)
+                    .flush(&memtable, self.epoch, covered_wal_entry_position, durable)
                     .await
             } else {
                 Box::pin(self.flusher.flush_with_indexes(
@@ -2696,6 +3288,7 @@ impl MemTableFlushHandler {
                     self.epoch,
                     &self.index_configs,
                     covered_wal_entry_position,
+                    durable,
                 ))
                 .await
             }
@@ -2725,15 +3318,15 @@ impl MemTableFlushHandler {
             // the read union until a later flush or WAL replay, else a transient
             // error reopens the hole.
             if flush_result.is_ok() {
-                let flushed_generation = memtable.generation();
+                let sstable = memtable.generation();
                 if self.grace.is_zero() {
                     state
                         .frozen_memtables
-                        .retain(|frozen| frozen.memtable.generation() != flushed_generation);
+                        .retain(|frozen| frozen.memtable.generation() != sstable);
                 } else {
                     let now = now_millis();
                     for frozen in state.frozen_memtables.iter_mut() {
-                        if frozen.memtable.generation() == flushed_generation {
+                        if frozen.memtable.generation() == sstable {
                             frozen.flushed_at_ms = Some(now);
                         }
                     }
@@ -2748,7 +3341,7 @@ impl MemTableFlushHandler {
 
         info!(
             "Flushed frozen memtable generation {} ({} rows in {:?})",
-            result.generation.generation,
+            result.sstable.generation,
             result.rows_flushed,
             start.elapsed()
         );
@@ -2935,11 +3528,7 @@ impl WriteStatsSnapshot {
 
     /// Get average WAL flush size in bytes.
     pub fn avg_wal_flush_bytes(&self) -> Option<u64> {
-        if self.wal_flush_count > 0 {
-            Some(self.wal_flush_bytes / self.wal_flush_count)
-        } else {
-            None
-        }
+        self.wal_flush_bytes.checked_div(self.wal_flush_count)
     }
 
     /// Get WAL write throughput (bytes per second based on WAL flush time).
@@ -2971,11 +3560,7 @@ impl WriteStatsSnapshot {
 
     /// Get average rows per index update.
     pub fn avg_index_update_rows(&self) -> Option<u64> {
-        if self.index_update_count > 0 {
-            Some(self.index_update_rows / self.index_update_count)
-        } else {
-            None
-        }
+        self.index_update_rows.checked_div(self.index_update_count)
     }
 
     /// Get average MemTable flush latency.
@@ -2989,11 +3574,8 @@ impl WriteStatsSnapshot {
 
     /// Get average MemTable flush size in rows.
     pub fn avg_memtable_flush_rows(&self) -> Option<u64> {
-        if self.memtable_flush_count > 0 {
-            Some(self.memtable_flush_rows / self.memtable_flush_count)
-        } else {
-            None
-        }
+        self.memtable_flush_rows
+            .checked_div(self.memtable_flush_count)
     }
 
     /// Log stats summary using tracing (for structured telemetry).
@@ -3052,6 +3634,7 @@ mod tests {
     use arrow_array::{Int32Array, StringArray};
     use arrow_schema::{DataType, Field};
     use lance_core::FenceReason;
+    use rstest::rstest;
     use tempfile::TempDir;
 
     async fn create_local_store() -> (Arc<ObjectStore>, Path, String, TempDir) {
@@ -3061,6 +3644,26 @@ mod tests {
         (store, path, uri, temp_dir)
     }
 
+    #[test]
+    fn test_merge_close_stage_preserves_first_error() {
+        let result = ShardWriter::merge_close_stage(
+            Err(Error::io("primary close error")),
+            "secondary close stage",
+            Err(Error::io("secondary close error")),
+        );
+
+        let error = result.expect_err("close must preserve the first error");
+        assert!(matches!(&error, Error::IO { .. }));
+        assert!(
+            error.to_string().contains("primary close error"),
+            "unexpected error: {error}"
+        );
+        assert!(
+            !error.to_string().contains("secondary close error"),
+            "secondary error replaced the primary error: {error}"
+        );
+    }
+
     /// Base schema with `id` marked as the unenforced primary key (delete needs
     /// a PK). `name` is nullable so a tombstone can null it.
     fn create_pk_test_schema() -> Arc<ArrowSchema> {
@@ -3322,7 +3925,7 @@ mod tests {
     /// await), but the tombstone still lands in the in-memory tier. The delete
     /// analog of `test_put_no_wait_non_durable_returns_no_watcher`.
     #[tokio::test]
-    async fn test_shard_writer_delete_no_wait_non_durable_returns_no_watcher() {
+    async fn test_non_durable_delete_is_read_your_writes() {
         let (store, base_path, base_uri, _temp) = create_local_store().await;
         let schema = create_pk_test_schema();
         let config = ShardWriterConfig {
@@ -3343,7 +3946,10 @@ mod tests {
             .delete_no_wait(vec![id_only_keys(&[2])])
             .await
             .unwrap();
-        assert!(watcher.is_none(), "non-durable delete has nothing to await");
+        // As with a put: a non-durable delete awaits its index apply, not an S3
+        // round-trip. It is read-your-writes, just not durable.
+        let mut watcher = watcher.expect("a non-durable delete awaits its index apply");
+        watcher.wait().await.unwrap();
 
         // Tombstone landed in the in-memory tier (5 rows + 1 tombstone).
         assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6);
@@ -3356,7 +3962,7 @@ mod tests {
     /// with an optional filter. Mirrors how a query reads a WAL table after a
     /// flush — the path the wallop fuzz exercised when it caught a deleted row
     /// resurfacing.
-    async fn read_flushed_ids_via_lsm(
+    async fn read_sstable_ids_via_lsm(
         writer: &ShardWriter,
         schema: Arc<ArrowSchema>,
         base_uri: &str,
@@ -3369,8 +3975,8 @@ mod tests {
         let manifest = writer.manifest().await.unwrap().unwrap();
         let mut snapshot =
             ShardSnapshot::new(shard_id).with_current_generation(manifest.current_generation);
-        for fg in &manifest.flushed_generations {
-            snapshot = snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        for sstable in &manifest.sstables {
+            snapshot = snapshot.with_sstable(sstable.generation, sstable.path.clone());
         }
         let mut scanner = LsmScanner::without_base_table(
             schema,
@@ -3406,14 +4012,13 @@ mod tests {
         ShardWriterConfig {
             shard_id,
             durable_write: false,
-            sync_indexed_write: true,
             manifest_scan_batch_size: 2,
             ..Default::default()
         }
     }
 
     /// Delete a key, then flush: the tombstone and the live row land in the
-    /// *same* flushed generation, so flush-time dedup must keep the tombstone
+    /// *same* SSTable, so flush-time dedup must keep the tombstone
     /// (newest) and the read must fold it away. Regression for the wallop
     /// phantom (deleted row resurfacing in a filtered read after flush).
     #[tokio::test]
@@ -3442,14 +4047,14 @@ mod tests {
         writer.wait_for_flush_drain().await.unwrap();
 
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
             vec![0, 1, 3, 4],
-            "id=2 deleted before flush; tombstone must not surface in a flushed-gen scan"
+            "id=2 deleted before flush; tombstone must not surface in an SSTable scan"
         );
         // The filtered read path (folds NOT _tombstone into the predicate) must
         // also drop it — this is the exact wallop failure shape (`id < 3`).
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3"))
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3"))
                 .await,
             vec![0, 1],
             "filtered read after flush must not resurface deleted id=2"
@@ -3463,7 +4068,7 @@ mod tests {
     /// mask the older row by PK. This is the wallop scenario (seed flushed,
     /// then delete, then flush).
     #[tokio::test]
-    async fn test_shard_writer_delete_across_flushed_generations() {
+    async fn test_shard_writer_delete_across_sstables() {
         let (store, base_path, base_uri, _temp) = create_local_store().await;
         let schema = create_pk_test_schema();
         let shard_id = Uuid::new_v4();
@@ -3492,12 +4097,12 @@ mod tests {
         writer.wait_for_flush_drain().await.unwrap();
 
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
             vec![1, 2, 3, 4],
             "id=0 tombstoned in a newer gen must mask the older gen's live row"
         );
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
                 .await,
             Vec::<i32>::new(),
             "filtered read 'id < 1' must not resurface cross-gen deleted id=0"
@@ -3506,13 +4111,13 @@ mod tests {
         writer.close().await.unwrap();
     }
 
-    /// Same as the cross-generation case, but the flushed generations carry a
+    /// Same as the cross-generation case, but the SSTables carry a
     /// BTree index on `id` (as every wallop table does). A filtered read
     /// `id < 1` resolves through the scalar index; the `NOT _tombstone` residual
     /// must still be applied or the deleted row leaks. This is the exact wallop
     /// failure (BTree id + `FilteredRead 'id < 1'` resurfacing deleted id=0).
     #[tokio::test]
-    async fn test_shard_writer_delete_across_flushed_generations_indexed() {
+    async fn test_shard_writer_delete_across_sstables_indexed() {
         let (store, base_path, base_uri, _temp) = create_local_store().await;
         let schema = create_pk_test_schema();
         let shard_id = Uuid::new_v4();
@@ -3544,12 +4149,12 @@ mod tests {
         writer.wait_for_flush_drain().await.unwrap();
 
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
             vec![1, 2, 3, 4],
             "indexed cross-gen: full scan must mask deleted id=0"
         );
         assert_eq!(
-            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
+            read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
                 .await,
             Vec::<i32>::new(),
             "indexed filtered read 'id < 1' must not resurface deleted id=0 (wallop repro)"
@@ -3731,9 +4336,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3774,9 +4378,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3803,7 +4406,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_put_no_wait_non_durable_returns_no_watcher() {
+    async fn test_non_durable_put_is_read_your_writes() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
@@ -3811,9 +4414,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3826,7 +4428,21 @@ mod tests {
         let batch = create_test_batch(&schema, 0, 10);
         let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap();
         assert_eq!(result.batch_positions, 0..1);
-        assert!(watcher.is_none(), "non-durable put has nothing to await");
+
+        // A non-durable put still has something to await: its *index apply*.
+        // Skipping durability now costs the caller durability only — not
+        // visibility. Before the index apply was split off the WAL flush, a
+        // non-durable write was not read-your-writes at all: the row stayed
+        // invisible until some later flush happened to index it.
+        let mut watcher = watcher.expect("a non-durable put awaits its index apply");
+        watcher.wait().await.unwrap();
+
+        let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap();
+        assert_eq!(
+            scanned.num_rows(),
+            10,
+            "a non-durable put must be readable as soon as it returns"
+        );
 
         let stats = writer.memtable_stats().await.unwrap();
         assert_eq!(stats.row_count, 10);
@@ -3843,9 +4459,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3878,9 +4493,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3914,12 +4528,12 @@ mod tests {
     }
 
     /// End-to-end check that the background flush handler rebuilds secondary
-    /// indexes on every flushed generation. Before this, the handler flushed
-    /// via plain `flush`, leaving flushed generations unindexed — point
+    /// indexes on every SSTable. Before this, the handler flushed
+    /// via plain `flush`, leaving SSTables unindexed — point
     /// lookups had to full-scan and vector search's index-only `fast_search`
     /// couldn't see the data at all.
     #[tokio::test]
-    async fn test_flushed_generation_is_indexed() {
+    async fn test_sstable_is_indexed() {
         use crate::index::DatasetIndexExt;
 
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
@@ -3930,9 +4544,8 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3964,22 +4577,18 @@ mod tests {
         writer.force_seal_active().await.unwrap();
         writer.wait_for_flush_drain().await.unwrap();
 
-        // Resolve the flushed generation recorded in the manifest.
+        // Resolve the SSTable recorded in the manifest.
         let manifest = writer.manifest().await.unwrap().unwrap();
-        assert_eq!(
-            manifest.flushed_generations.len(),
-            1,
-            "expected exactly one flushed generation"
-        );
+        assert_eq!(manifest.sstables.len(), 1, "expected exactly one SSTable");
         let gen_uri = format!(
             "{}/_mem_wal/{}/{}",
-            base_uri, shard_id, manifest.flushed_generations[0].path
+            base_uri, shard_id, manifest.sstables[0].path
         );
 
-        // The flushed generation must carry the BTree index built during flush.
+        // The SSTable must carry the BTree index built during flush.
         let dataset = crate::Dataset::open(&gen_uri).await.unwrap();
         let indices = dataset.load_indices().await.unwrap();
-        assert_eq!(indices.len(), 1, "flushed generation should have one index");
+        assert_eq!(indices.len(), 1, "SSTable should have one index");
         assert_eq!(indices[0].name, "id_idx");
 
         // A PK filter over it must resolve through the index, not a full scan.
@@ -4019,9 +4628,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 1024, // Very small - will trigger flush quickly
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4110,7 +4718,113 @@ mod tests {
             call_count.load(Ordering::SeqCst)
         );
 
-        executor.shutdown_all().await.ok();
+        executor
+            .shutdown_all()
+            .await
+            .expect("dispatcher should shut down successfully");
+    }
+
+    #[tokio::test]
+    async fn test_task_executor_shutdown_propagates_cleanup_error_and_joins_all_tasks() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+
+        struct CleanupHandler {
+            cleanup_count: Arc<AtomicUsize>,
+            error_message: Option<&'static str>,
+        }
+
+        #[async_trait]
+        impl MessageHandler<u32> for CleanupHandler {
+            async fn handle(&mut self, _message: u32) -> Result<()> {
+                Ok(())
+            }
+
+            async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> {
+                self.cleanup_count.fetch_add(1, Ordering::SeqCst);
+                match self.error_message {
+                    Some(message) => Err(Error::io(message)),
+                    None => Ok(()),
+                }
+            }
+        }
+
+        let executor = TaskExecutor::new();
+        let cleanup_count = Arc::new(AtomicUsize::new(0));
+        let (_failing_tx, failing_rx) = mpsc::unbounded_channel::<u32>();
+        executor
+            .add_handler(
+                "failing-cleanup".to_string(),
+                Box::new(CleanupHandler {
+                    cleanup_count: cleanup_count.clone(),
+                    error_message: Some("intentional cleanup failure"),
+                }),
+                failing_rx,
+            )
+            .unwrap();
+        let (_successful_tx, successful_rx) = mpsc::unbounded_channel::<u32>();
+        executor
+            .add_handler(
+                "successful-cleanup".to_string(),
+                Box::new(CleanupHandler {
+                    cleanup_count: cleanup_count.clone(),
+                    error_message: None,
+                }),
+                successful_rx,
+            )
+            .unwrap();
+
+        let error = executor
+            .shutdown_all()
+            .await
+            .expect_err("shutdown must propagate the handler cleanup failure");
+        assert!(matches!(&error, Error::IO { .. }));
+        assert!(
+            error.to_string().contains("intentional cleanup failure"),
+            "unexpected error: {error}"
+        );
+        assert_eq!(
+            cleanup_count.load(Ordering::SeqCst),
+            2,
+            "shutdown must join and clean up every task after the first failure"
+        );
+        assert!(executor.tasks.read().unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_task_executor_shutdown_propagates_task_panic() {
+        struct PanickingCleanupHandler;
+
+        #[async_trait]
+        impl MessageHandler<u32> for PanickingCleanupHandler {
+            async fn handle(&mut self, _message: u32) -> Result<()> {
+                Ok(())
+            }
+
+            async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> {
+                panic!("intentional cleanup panic");
+            }
+        }
+
+        let executor = TaskExecutor::new();
+        let (_tx, rx) = mpsc::unbounded_channel::<u32>();
+        executor
+            .add_handler(
+                "panicking-cleanup".to_string(),
+                Box::new(PanickingCleanupHandler),
+                rx,
+            )
+            .unwrap();
+
+        let error = executor
+            .shutdown_all()
+            .await
+            .expect_err("shutdown must propagate the task panic");
+        assert!(matches!(&error, Error::Internal { .. }));
+        assert!(
+            error.to_string().contains("panicking-cleanup")
+                && error.to_string().contains("panicked during shutdown"),
+            "unexpected error: {error}"
+        );
     }
 
     /// Same as the local-fs test but against memory:// — closer to S3
@@ -4126,9 +4840,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4171,9 +4884,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold — every batch crosses it.
             max_memtable_size: 64,
             manifest_scan_batch_size: 2,
@@ -4238,9 +4950,8 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: usize::MAX,
             max_unflushed_memtable_bytes: usize::MAX,
             manifest_scan_batch_size: 2,
@@ -4305,6 +5016,55 @@ mod tests {
         reopened.close().await.unwrap();
     }
 
+    #[rstest]
+    #[case::memtable(true)]
+    #[case::wal_only(false)]
+    #[tokio::test]
+    async fn test_close_propagates_final_wal_persistence_failure(#[case] enable_memtable: bool) {
+        let (store, base_path, controls) = failing_memory_store().await;
+        let base_uri = "memory:///";
+        let schema = create_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            enable_memtable,
+            max_wal_buffer_size: usize::MAX,
+            max_wal_flush_interval: None,
+            max_wal_persist_retries: 0,
+            max_memtable_size: usize::MAX,
+            max_unflushed_memtable_bytes: usize::MAX,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+        let task_executor = writer.task_executor.clone();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        controls.fail_wal_puts(usize::MAX);
+
+        let error = writer
+            .close()
+            .await
+            .expect_err("close must propagate the final WAL persistence failure");
+        assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(
+            error
+                .to_string()
+                .contains("injected transient WAL put failure"),
+            "unexpected error: {error}"
+        );
+        assert!(
+            task_executor.tasks.read().unwrap().is_empty(),
+            "close must join background tasks before returning an error"
+        );
+    }
+
     /// Regression: the memtable flush should successfully fire many
     /// times in a row. A bug where every flush wrote the same path was
     /// caught by lance-format/lance#6713.
@@ -4319,9 +5079,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold so a few batches cross it.
             max_memtable_size: 1024,
             manifest_scan_batch_size: 2,
@@ -4462,7 +5221,7 @@ mod tests {
             durable_write: true,
             enable_memtable: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             manifest_scan_batch_size: 2,
             ..Default::default()
         }
@@ -4487,8 +5246,9 @@ mod tests {
         .await
         .unwrap();
 
-        // Two durable puts → two WAL entries (durable_write triggers an
-        // explicit flush per put).
+        // Two durable puts → two WAL entries. Each `put` awaits its own append
+        // (driven by the background ticker) before returning, so the second
+        // batch is only pushed after the first is durable — they never coalesce.
         let r1 = writer
             .put(vec![create_test_batch(&schema, 0, 4)])
             .await
@@ -4519,6 +5279,51 @@ mod tests {
         assert!(e0.writer_epoch >= 1);
     }
 
+    /// A durable WAL-only put is driven by the background ticker, not an inline
+    /// per-put flush: `put().await` must not return until the ticker's append
+    /// advances the durability watermark it waits on. With a long interval and a
+    /// buffer the write cannot cross, the ticker is the *only* thing that can
+    /// make the put durable — so if the WAL entry is present the instant `put`
+    /// returns (before any `close()`), the watermark-wait is doing its job.
+    #[tokio::test]
+    async fn test_wal_only_durable_put_waits_for_ticker_append() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+        let shard_id = Uuid::new_v4();
+
+        let mut config = wal_only_config(shard_id);
+        config.max_wal_flush_interval = Some(Duration::from_millis(100));
+        config.max_wal_buffer_size = 100 * 1024 * 1024; // never crossed
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 4)])
+            .await
+            .unwrap();
+
+        // `put` returned, so the batch must already be durable — read the WAL
+        // directly, without closing the writer.
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        assert_eq!(tailer.next_position().await.unwrap(), 2);
+        let entry = tailer.read_entry(1).await.unwrap().unwrap();
+        assert_eq!(entry.batches.len(), 1);
+        assert_eq!(entry.batches[0].num_rows(), 4);
+
+        writer.close().await.unwrap();
+    }
+
     #[tokio::test]
     async fn test_wal_only_rejects_index_configs() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
@@ -4753,9 +5558,8 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4779,8 +5583,9 @@ mod tests {
     }
 
     // A durable write whose WAL PUT keeps failing poisons the writer with a
-    // typed persistence failure; the next write fails fast with the same reason;
-    // and once storage heals, reopening replays the WAL and writes resume.
+    // typed persistence failure; the next write *and every read* fail fast with
+    // the same reason; and once storage heals, reopening replays the WAL and
+    // writes resume.
     #[tokio::test]
     async fn test_writer_poisons_on_persistence_failure_and_recovers_on_reopen() {
         let (store, base_path, controls) = failing_memory_store().await;
@@ -4819,6 +5624,31 @@ mod tests {
             .await
             .unwrap_err();
         assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // ...and rejects *reads* too. Batch 0 was committed to the BatchStore
+        // before its WAL PUT failed, so a poisoned writer that still served
+        // reads would hand out a row that is not durable and that replay will
+        // not reproduce — a divergent snapshot. Mirrors SlateDB's
+        // `check_closed()` at the top of every read.
+        for reason in [
+            writer.scan().await.err().and_then(|e| e.fence_reason()),
+            writer
+                .active_memtable_ref()
+                .await
+                .err()
+                .and_then(|e| e.fence_reason()),
+            writer
+                .in_memory_memtable_refs()
+                .await
+                .err()
+                .and_then(|e| e.fence_reason()),
+        ] {
+            assert_eq!(reason, Some(FenceReason::PersistenceFailure));
+        }
+
+        // Stats stay readable: this is what an operator (and the eviction path)
+        // inspects to decide what to do about the poisoned shard.
+        writer.memtable_stats().await.unwrap();
         drop(writer);
 
         // Storage heals: reopening replays the WAL and accepts writes again.
@@ -4832,71 +5662,714 @@ mod tests {
             .unwrap();
     }
 
-    /// Replay-on-open recovers durable WAL entries that were never flushed
-    /// to a Lance generation. Setup: writer A durably writes batches, drops
-    /// without close (so MemTable freeze never runs); writer B reopens and
-    /// must see A's rows in its MemTable scan.
+    /// A doomed open must fail on local validation *before* it claims the epoch,
+    /// so it cannot fence the healthy writer already serving the shard. The
+    /// index-config check used to run *after* `claim_epoch` (and, for a successor,
+    /// after `write_fence_sentinel`), so a rejected open still bumped the stored
+    /// epoch and fenced the incumbent.
     #[tokio::test]
-    async fn test_memtable_replay_recovers_unflushed_writes() {
+    async fn test_doomed_open_does_not_fence_incumbent() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = schema_with_pk();
         let shard_id = Uuid::new_v4();
 
-        // Writer A: write two durable batches, then drop without close.
-        // The WAL files persist; the in-memory MemTable does not.
-        {
-            let writer_a = ShardWriter::open(
-                store.clone(),
-                base_path.clone(),
-                base_uri.clone(),
-                memtable_config_with_pk(shard_id),
-                schema.clone(),
-                vec![],
-            )
-            .await
-            .unwrap();
-            writer_a
-                .put(vec![create_test_batch(&schema, 0, 5)])
-                .await
-                .unwrap();
-            writer_a
-                .put(vec![create_test_batch(&schema, 100, 3)])
-                .await
-                .unwrap();
-            // intentionally drop without close()
-        }
-
-        // Writer B reopens. Replay must rehydrate A's two batches into the
-        // active MemTable.
-        let writer_b = ShardWriter::open(
-            store,
-            base_path,
-            base_uri,
+        let writer_a = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
             memtable_config_with_pk(shard_id),
-            schema,
+            schema.clone(),
             vec![],
         )
         .await
         .unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 0, 1)])
+            .await
+            .unwrap();
 
-        let stats = writer_b.memtable_stats().await.unwrap();
-        assert_eq!(
-            stats.row_count, 8,
-            "expected replay to insert 5 + 3 = 8 rows, got {}",
-            stats.row_count
-        );
-        assert_eq!(
-            stats.batch_count, 2,
-            "expected replay to insert 2 batches, got {}",
-            stats.batch_count
+        // An index config that disagrees with the schema (FTS on the Int32 `id`
+        // column) is rejected on local validation. On the old path this rejection
+        // landed only after the epoch had already been claimed.
+        let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new(
+            "bad_fts".to_string(),
+            0,
+            "id".to_string(),
+        ));
+        let err = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![bad_fts],
+        )
+        .await
+        .map(|_| ())
+        .expect_err("an FTS index on a non-Utf8 column must be rejected");
+        assert!(
+            err.to_string().contains("bad_fts") && err.to_string().contains("Utf8"),
+            "unexpected error: {err}"
         );
 
-        writer_b.close().await.unwrap();
+        // The incumbent is untouched: not fenced, still accepting writes.
+        writer_a.check_fenced().await.unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 1, 1)])
+            .await
+            .unwrap();
+        writer_a.close().await.unwrap();
     }
 
-    /// Replay is a no-op on a fresh shard: the MemTable starts empty.
+    /// A failed dispatch during `freeze_memtable` must not drop the outgoing
+    /// table's rows from the read view. The active memtable is replaced before
+    /// the WAL-flush and index-apply sends; a send that failed (background tasks
+    /// gone) used to return before the outgoing table was retained in
+    /// `frozen_memtables`, so its accepted rows silently vanished — a scan
+    /// returned 0 rows with no error. The writer must instead retain the table
+    /// and poison, so reads fail fast rather than serve a divergent snapshot.
     #[tokio::test]
-    async fn test_memtable_replay_no_op_on_fresh_shard() {
+    async fn test_freeze_dispatch_failure_retains_rows_and_poisons() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Non-durable + no ticker: the put is read-your-writes (waits for its
+        // index apply) but nothing is WAL-flushed, so the freeze below still owes
+        // a WAL append.
+        let config = ShardWriterConfig {
+            durable_write: false,
+            max_wal_flush_interval: None,
+            ..memtable_config_with_pk(shard_id)
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        assert_eq!(writer.memtable_stats().await.unwrap().row_count, 10);
+
+        // Tear the background tasks down out from under the writer, so the
+        // freeze's dispatch sends hit closed channels.
+        writer.abort().await.unwrap();
+
+        let err = writer
+            .force_seal_active()
+            .await
+            .expect_err("force_seal_active must surface the failed dispatch");
+        assert!(
+            err.to_string().contains("channel closed"),
+            "unexpected error: {err}"
+        );
+
+        // The failure poisoned the writer: reads fail fast instead of returning a
+        // silent zero-row snapshot of a shard whose rows were dropped.
+        assert!(
+            writer.scan().await.is_err(),
+            "a poisoned writer must reject reads, not serve a divergent snapshot"
+        );
+        assert!(writer.in_memory_memtable_refs().await.is_err());
+    }
+
+    /// A WAL holding more batches than one memtable's capacity must reopen.
+    ///
+    /// One memtable holds at most `max_memtable_batches` batches, but a WAL is
+    /// unbounded, so replay has to rotate — seal the full memtable, start a fresh
+    /// one — exactly as the live write path does. Before, replay stuffed
+    /// everything into a single memtable and `open()` failed outright with
+    /// "MemTable batch store is full", leaving the shard permanently unopenable.
+    #[tokio::test]
+    async fn test_replay_rotates_when_wal_exceeds_one_memtable() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        const N: i32 = 8;
+
+        // Writer A has a *large* capacity, so its eight one-batch puts all land in
+        // a single memtable and it never freezes or flushes a generation of its
+        // own. Dropping it without close leaves an eight-entry WAL and no
+        // generations — a WAL that no single small memtable could hold.
+        let writer_a_config = ShardWriterConfig {
+            max_memtable_batches: 1000,
+            ..memtable_config_with_pk(shard_id)
+        };
+        // Writer B has a *two-batch* capacity, so replaying that eight-entry WAL is
+        // exactly what must rotate. Keeping the configs distinct isolates replay
+        // rotation from the live rotation writer A would otherwise do concurrently.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        {
+            let writer_a = ShardWriter::open(
+                store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                writer_a_config,
+                schema.clone(),
+                vec![],
+            )
+            .await
+            .unwrap();
+            for id in 0..N {
+                writer_a
+                    .put(vec![create_test_batch(&schema, id, 1)])
+                    .await
+                    .unwrap();
+            }
+            // Drop without close: only the WAL survives, and it holds more batches
+            // than writer B's memtable can.
+        }
+
+        // Total rows across the active memtable plus every SSTable.
+        // Distinct ids, so no cross-generation dedup — a plain sum is exact.
+        async fn total_rows(writer: &ShardWriter, base_uri: &str, shard_id: Uuid) -> usize {
+            let mut rows = writer.memtable_stats().await.unwrap().row_count;
+            let manifest = writer.manifest().await.unwrap().unwrap();
+            for sstable in &manifest.sstables {
+                let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, sstable.path);
+                let dataset = crate::Dataset::open(&gen_uri).await.unwrap();
+                rows += dataset.count_rows(None).await.unwrap();
+            }
+            rows
+        }
+
+        // Reopen. This used to fail with a full-batch-store error.
+        let writer_b = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            config.clone(),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .expect("a WAL larger than one memtable must still reopen");
+
+        // Rotation produced sealed memtables, and replay flushed each to a Lance
+        // generation rather than holding it in memory or leaving it in the WAL.
+        let manifest = writer_b.manifest().await.unwrap().unwrap();
+        assert!(
+            !manifest.sstables.is_empty(),
+            "replay must have sealed and flushed at least one full memtable"
+        );
+
+        // Every row survived, split between the SSTables and the
+        // active (partial) memtable.
+        assert_eq!(
+            total_rows(&writer_b, &base_uri, shard_id).await as i32,
+            N,
+            "every replayed row must be durable, across generations and the active memtable"
+        );
+        writer_b.close().await.unwrap();
+
+        // Because the sealed memtables were flushed, the manifest's replay cursor
+        // advanced past their WAL entries — so a second reopen replays only the
+        // tail and still accounts for every row. The WAL truncates across reopens
+        // rather than growing without bound.
+        let writer_c =
+            ShardWriter::open(store, base_path, base_uri.clone(), config, schema, vec![])
+                .await
+                .unwrap();
+        assert_eq!(total_rows(&writer_c, &base_uri, shard_id).await as i32, N);
+        writer_c.close().await.unwrap();
+    }
+
+    /// Replay-on-open recovers durable WAL entries that were never flushed
+    /// to a Lance generation. Setup: writer A durably writes batches, drops
+    /// without close (so MemTable freeze never runs); writer B reopens and
+    /// must see A's rows in its MemTable scan.
+    #[tokio::test]
+    async fn test_memtable_replay_recovers_unflushed_writes() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Writer A: write two durable batches, then drop without close.
+        // The WAL files persist; the in-memory MemTable does not.
+        {
+            let writer_a = ShardWriter::open(
+                store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                memtable_config_with_pk(shard_id),
+                schema.clone(),
+                vec![],
+            )
+            .await
+            .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 0, 5)])
+                .await
+                .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 100, 3)])
+                .await
+                .unwrap();
+            // intentionally drop without close()
+        }
+
+        // Writer B reopens. Replay must rehydrate A's two batches into the
+        // active MemTable.
+        let writer_b = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema,
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        let stats = writer_b.memtable_stats().await.unwrap();
+        assert_eq!(
+            stats.row_count, 8,
+            "expected replay to insert 5 + 3 = 8 rows, got {}",
+            stats.row_count
+        );
+        assert_eq!(
+            stats.batch_count, 2,
+            "expected replay to insert 2 batches, got {}",
+            stats.batch_count
+        );
+
+        writer_b.close().await.unwrap();
+    }
+
+    /// Replayed batches are already WAL-durable, so the first flush after a
+    /// reopen must not re-append them to the WAL or re-insert them into the
+    /// indexes. Before replay stamped the durability cursor it stayed at
+    /// "nothing flushed", so the next flush re-covered `[0, end)`: it appended
+    /// the already-durable rows a second time *and* re-indexed them. None of
+    /// the in-memory indexes is idempotent, so an indexed PK lookup returned
+    /// the row twice while a full scan still looked healthy.
+    #[tokio::test]
+    async fn test_replay_does_not_reappend_or_reindex() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Writer A: two durable batches (5 + 3 = 8 rows), dropped without close.
+        {
+            let writer_a = ShardWriter::open(
+                store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                memtable_config_with_pk(shard_id),
+                schema.clone(),
+                vec![],
+            )
+            .await
+            .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 0, 5)])
+                .await
+                .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 100, 3)])
+                .await
+                .unwrap();
+        }
+
+        // Writer B reopens and replays A's two batches.
+        let writer_b = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        let stats = writer_b.memtable_stats().await.unwrap();
+        assert_eq!(stats.batch_count, 2);
+        assert_eq!(
+            stats.durable_batch_count, 2,
+            "replayed batches came from the WAL, so the durability cursor must already cover them"
+        );
+
+        // One more durable put. Its flush must cover only the new batch.
+        writer_b
+            .put(vec![create_test_batch(&schema, 200, 2)])
+            .await
+            .unwrap();
+
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut wal_rows = 0;
+        for position in first..next {
+            if let Some(entry) = tailer.read_entry(position).await.unwrap() {
+                wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::<usize>();
+            }
+        }
+        assert_eq!(
+            wal_rows, 10,
+            "WAL must hold 8 replayed + 2 new rows; a re-covering flush re-appends the replayed 8"
+        );
+
+        // The indexed arm must not see a replayed row twice.
+        let mut scanner = writer_b.scan().await.unwrap();
+        scanner.filter("id = 0").unwrap();
+        let hit = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            hit.num_rows(),
+            1,
+            "indexed PK lookup returned the replayed row more than once"
+        );
+
+        let all = writer_b
+            .scan()
+            .await
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(all.num_rows(), 10);
+
+        writer_b.close().await.unwrap();
+    }
+
+    /// A non-durable put is readable through the **index-backed** arms the moment
+    /// it returns, not just through a full scan.
+    ///
+    /// This is what splitting the index apply off the WAL flush buys. Before, the
+    /// index apply only ran as one arm of the flush, so with `durable_write:
+    /// false` nothing triggered it on the put path at all: the row sat in the
+    /// batch store, unindexed, until some later flush happened along. A full scan
+    /// (which reads the batch store directly) could still find it, while every
+    /// index-accelerated query could not — the tiers disagreed. Now the apply is
+    /// triggered per-put in both modes, so `put` returning means "indexed".
+    #[tokio::test]
+    async fn test_non_durable_put_is_visible_through_the_index() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        let config = ShardWriterConfig {
+            durable_write: false,
+            ..memtable_config_with_pk(shard_id)
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+
+        // The indexed PK lookup must find it — this is the arm that saw nothing
+        // before, because the index apply had never run.
+        let mut scanner = writer.scan().await.unwrap();
+        scanner.filter("id = 3").unwrap();
+        let hit = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            hit.num_rows(),
+            1,
+            "an index-backed lookup must see a non-durable put as soon as it returns"
+        );
+
+        // ...and so must the unindexed full scan, i.e. the tiers agree.
+        let all = writer.scan().await.unwrap().try_into_batch().await.unwrap();
+        assert_eq!(all.num_rows(), 5);
+
+        writer.close().await.unwrap();
+    }
+
+    /// The durability cursor is writer-global, so a put into a *post-rotation*
+    /// memtable must still wait for its own WAL append.
+    ///
+    /// Batch positions restart at 0 in every memtable, but the durability watch
+    /// channel spans the writer's whole life and is never reset. When the put
+    /// path targeted a memtable-local position, the first N puts into every
+    /// memtable after the first were already "satisfied" by the *previous*
+    /// memtable's N appends: they acked instantly, with no WAL append, and
+    /// `durable_write: true` silently degraded to non-durable. Worse, the next
+    /// append then sent a *smaller* value, walking the watermark backwards and
+    /// hanging any watcher still waiting on the old, higher one.
+    ///
+    /// The cursor is now a writer-global exclusive count and every target is
+    /// lifted through the store's `global_offset`, so it only ever moves forward
+    /// and a post-rotation put can only be acked by its own append.
+    #[tokio::test]
+    async fn test_durable_ack_after_rotation_requires_its_own_wal_append() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // A two-batch memtable, so the third put forces a freeze + rotation.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // Fill and rotate the first memtable.
+        for i in 0..3 {
+            writer
+                .put(vec![create_test_batch(&schema, i * 5, 5)])
+                .await
+                .unwrap();
+        }
+        let stats = writer.memtable_stats().await.unwrap();
+        assert!(
+            stats.global_offset > 0,
+            "expected a rotation; the active memtable is still the writer's first"
+        );
+
+        // Every batch acked so far must be genuinely durable, and the cursor must
+        // cover the active memtable's entire prefix rather than lagging inside it.
+        let stats = writer.memtable_stats().await.unwrap();
+        assert!(
+            stats.durable_batch_count >= stats.global_offset + stats.batch_count,
+            "durable_write acked a put the WAL never received: durable={} but the active \
+             memtable spans [{}, {})",
+            stats.durable_batch_count,
+            stats.global_offset,
+            stats.global_offset + stats.batch_count
+        );
+
+        // And the WAL really holds every row we acked (3 puts x 5 rows).
+        writer.close().await.unwrap();
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut wal_rows = 0;
+        for position in first..next {
+            if let Some(entry) = tailer.read_entry(position).await.unwrap() {
+                wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::<usize>();
+            }
+        }
+        assert_eq!(
+            wal_rows, 15,
+            "every acked row must be in the WAL; a post-rotation put that acked without an \
+             append would leave rows missing"
+        );
+    }
+
+    /// A background tick must append the **oldest** store that still owes the WAL,
+    /// never "whatever memtable is active".
+    ///
+    /// A tick carries no store: it is enqueued by a timer and resolved when it is
+    /// handled. So a tick enqueued before a freeze is handled *after* it, and
+    /// resolving to the active memtable would append the incoming memtable's
+    /// batches ahead of the outgoing memtable's tail. WAL entry positions are
+    /// assigned in append-call order, replay walks them ascending, row positions
+    /// follow, and primary-key recency is "newest visible row position wins" — so
+    /// that inverts dedup after a crash, handing the key to the stale row. It
+    /// survives the crash that caused it, and a full scan cannot see it.
+    #[test]
+    fn test_next_pending_store_picks_the_oldest_owing_an_append() {
+        let schema = create_test_schema();
+
+        // A frozen store of 2 batches at coordinate 0, and the active store that
+        // rotated in behind it at coordinate 2.
+        let frozen = Arc::new(BatchStore::with_capacity(4));
+        frozen.append(create_test_batch(&schema, 0, 1)).unwrap();
+        frozen.append(create_test_batch(&schema, 1, 1)).unwrap();
+        let active = Arc::new(BatchStore::with_capacity_at(4, 2));
+        active.append(create_test_batch(&schema, 2, 1)).unwrap();
+
+        let frozen_list = || std::iter::once(Arc::clone(&frozen));
+
+        // Nothing durable: the frozen store owes the oldest append, so it wins —
+        // even though the active memtable also has un-appended batches.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 0).unwrap();
+        assert!(
+            Arc::ptr_eq(&picked, &frozen),
+            "the outgoing memtable's tail must be appended before the incoming one's head"
+        );
+
+        // Still true partway through the frozen store.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 1).unwrap();
+        assert!(Arc::ptr_eq(&picked, &frozen));
+
+        // Once the frozen store is fully durable, the active one is next.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 2).unwrap();
+        assert!(Arc::ptr_eq(&picked, &active));
+
+        // Everything durable: nothing to do.
+        assert!(next_pending_store(frozen_list(), Arc::clone(&active), 3).is_none());
+    }
+
+    /// A durable writer with no flush ticker cannot make progress in either
+    /// mode — the ticker is the only thing that drives the WAL append the put
+    /// waits on — so `open()` rejects it rather than letting a put block forever.
+    #[rstest]
+    #[case::memtable(true)]
+    #[case::wal_only(false)]
+    #[tokio::test]
+    async fn test_open_rejects_durable_write_without_a_ticker(#[case] enable_memtable: bool) {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        let config = ShardWriterConfig {
+            durable_write: true,
+            enable_memtable,
+            max_wal_flush_interval: None,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let Err(err) = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]).await
+        else {
+            panic!("durable_write with no ticker must be rejected");
+        };
+        assert!(
+            err.to_string().contains("max_wal_flush_interval"),
+            "the error must name the knob, got: {err}"
+        );
+    }
+
+    /// WAL entries must be appended in global batch-position order across a
+    /// memtable rotation, because append order *is* primary-key recency order.
+    ///
+    /// `WalAppender::append` assigns each entry's position from its own counter,
+    /// in call order. Replay walks those positions ascending and assigns row
+    /// positions in that order. Primary-key recency is "newest visible row
+    /// position wins". So an out-of-order append silently inverts dedup after a
+    /// crash — the stale row wins — and a full scan cannot see it.
+    ///
+    /// The hazard is the background ticker. If it resolved to "whatever memtable
+    /// is active" rather than to the oldest store still owing an append, a tick
+    /// enqueued before a freeze but handled after it would append the *incoming*
+    /// memtable's batches ahead of the outgoing memtable's tail. So the target is
+    /// resolved from the durability cursor, not from wall-clock timing.
+    #[tokio::test]
+    async fn test_wal_append_order_preserves_pk_recency_across_rotation() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Two batches per memtable, so the second put fills it and rotates.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // Memtable 1: ids 7 then 20 (the second fills it and triggers the freeze).
+        writer
+            .put(vec![create_test_batch(&schema, 7, 1)])
+            .await
+            .unwrap();
+        writer
+            .put(vec![create_test_batch(&schema, 20, 1)])
+            .await
+            .unwrap();
+        // Memtable 2 overwrites id=7 with a newer row. Its append must land in the
+        // WAL *after* memtable 1's, or replay would resolve id=7 to the stale copy.
+        writer
+            .put(vec![create_test_batch(&schema, 30, 1)])
+            .await
+            .unwrap();
+        writer.close().await.unwrap();
+
+        // Walk the WAL in entry order and collect the ids as replay would see them.
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut ids: Vec<i32> = Vec::new();
+        for position in first..next {
+            let Some(entry) = tailer.read_entry(position).await.unwrap() else {
+                continue;
+            };
+            for batch in &entry.batches {
+                let column = batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap();
+                ids.extend((0..column.len()).map(|i| column.value(i)));
+            }
+        }
+
+        assert_eq!(
+            ids,
+            vec![7, 20, 30],
+            "WAL entries must follow global batch-position order; memtable 1's rows must \
+             precede memtable 2's, or replay inverts primary-key recency"
+        );
+    }
+
+    /// An index config that disagrees with the schema fails `open()` outright.
+    /// It must not be allowed to accept writes: the insert would fail
+    /// deterministically on every batch, including batches replayed from the
+    /// WAL, so once a row was durable the shard could never reopen. Before this
+    /// check, an FTS index on a non-Utf8 column silently indexed nothing and the
+    /// shard reported healthy.
+    #[tokio::test]
+    async fn test_open_rejects_index_config_that_disagrees_with_schema() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // `id` is Int32, not a string column.
+        let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new(
+            "bad_fts".to_string(),
+            0,
+            "id".to_string(),
+        ));
+
+        let Err(err) = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema,
+            vec![bad_fts],
+        )
+        .await
+        else {
+            panic!("open must reject an FTS index on a non-Utf8 column");
+        };
+
+        let message = err.to_string();
+        assert!(
+            message.contains("bad_fts") && message.contains("Utf8"),
+            "error must name the index and the constraint, got: {message}"
+        );
+    }
+
+    /// Replay is a no-op on a fresh shard: the MemTable starts empty.
+    #[tokio::test]
+    async fn test_memtable_replay_no_op_on_fresh_shard() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = schema_with_pk();
         let shard_id = Uuid::new_v4();
@@ -4919,13 +6392,13 @@ mod tests {
 
     /// Regression for the OSS-WAL compactor-drain bug: after a flush
     /// records its generation in the manifest and an external compactor
-    /// later drains `flushed_generations` back to empty (the legitimate
-    /// outcome of merging the generation into the base table), reopening
+    /// later drains `sstables` back to empty (the legitimate
+    /// outcome after compacting the SSTable into the base table), reopening
     /// the writer must not re-replay the already-flushed WAL entry into
     /// the active memtable.
     ///
     /// Under the pre-fix logic, replay disambiguated "fresh shard" from
-    /// "flushed-then-compacted" with `flushed_generations.is_empty()`,
+    /// "flushed-then-compacted" with `sstables.is_empty()`,
     /// which collapsed both cases into start-at-0. With 1-based WAL
     /// positions and a default cursor of 0 meaning "no flush stamped",
     /// the flush-then-drain sequence leaves `replay_after_wal_entry_position`
@@ -4939,7 +6412,7 @@ mod tests {
         let shard_id = Uuid::new_v4();
 
         // Writer A: write 5 rows, close (forces a flush of the active
-        // memtable). The manifest now records a flushed generation and
+        // memtable). The manifest now records an SSTable and
         // pins `replay_after_wal_entry_position` to the covered WAL entry.
         {
             let writer_a = ShardWriter::open(
@@ -4959,14 +6432,14 @@ mod tests {
             writer_a.close().await.unwrap();
         }
 
-        // Simulate an external compactor merging the flushed generation
-        // into the base table: drain `flushed_generations` to empty via a
+        // Simulate an external compactor compacting the SSTable.
+        // into the base table: drain `sstables` to empty via a
         // direct manifest commit. The cursor stays where the flush put it.
         let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2);
         let pre = manifest_store.read_latest().await.unwrap().unwrap();
         assert!(
-            !pre.flushed_generations.is_empty(),
-            "writer A's close() should have stamped a flushed generation"
+            !pre.sstables.is_empty(),
+            "writer A's close() should have stamped an SSTable"
         );
         let cursor_at_flush = pre.replay_after_wal_entry_position;
         assert!(
@@ -4980,22 +6453,22 @@ mod tests {
         manifest_store
             .commit_update(compactor_epoch, |current| ShardManifest {
                 version: current.version + 1,
-                flushed_generations: vec![],
+                sstables: vec![],
                 ..current.clone()
             })
             .await
             .unwrap();
         let post = manifest_store.read_latest().await.unwrap().unwrap();
         assert!(
-            post.flushed_generations.is_empty(),
-            "compactor drain should have left flushed_generations empty"
+            post.sstables.is_empty(),
+            "compactor drain should have left sstables empty"
         );
         assert_eq!(
             post.replay_after_wal_entry_position, cursor_at_flush,
             "compactor must not touch the replay cursor"
         );
 
-        // Writer B reopens. Pre-fix: replay saw flushed_generations empty,
+        // Writer B reopens. Pre-fix: replay saw sstables empty,
         // restarted at WAL position 0, and re-inserted writer A's rows.
         // Post-fix: replay starts at cursor + 1, finds no entry, and the
         // memtable stays empty.
@@ -5230,8 +6703,9 @@ mod tests {
     /// fence), the drained batches were dropped — the next concurrent put
     /// would then see an empty pending queue and spuriously return Ok,
     /// hiding the data loss. With the snapshot/commit fix, the failed flush
-    /// leaves the batches in the queue, and the concurrent put gets a clean
-    /// fence error too (when its own flush attempts the same WAL position).
+    /// leaves the batches in the queue for retry, and — because the flush is
+    /// terminal — it poisons the writer, waking *both* parked durability
+    /// waiters with the typed fence error instead of either one hanging.
     #[tokio::test]
     async fn test_wal_only_fenced_concurrent_puts_do_not_silently_succeed() {
         use std::sync::Arc;
@@ -5282,9 +6756,10 @@ mod tests {
         // the destructive-drain bug, the first flush would consume both
         // pending batches into a failing append; the second flush would
         // see an empty queue and return spurious success, silently losing
-        // the second put's data. With the snapshot/commit fix, the failed
-        // append leaves both batches in the queue and the second flush
-        // also fails with the fence error.
+        // the second put's data. Now both puts park on the durability
+        // watermark; the ticker's append fails with the fence, poisons the
+        // writer, and both waiters wake with the fence error — batches intact
+        // in the queue.
         let a1 = writer_a.clone();
         let a2 = writer_a.clone();
         let schema1 = schema.clone();
@@ -5342,6 +6817,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_memtable_stats_record_index_update() {
+        // MemTable mode with a BTree index: index application runs on its own
+        // task and must record an index-update stat. Regression for the stat
+        // silently reading zero after index apply moved off the WAL-flush path.
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let index_configs = vec![MemIndexConfig::BTree(BTreeIndexConfig {
+            name: "id_idx".to_string(),
+            field_id: 0,
+            column: "id".to_string(),
+        })];
+
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            flush_test_config(Uuid::new_v4()),
+            schema.clone(),
+            index_configs,
+        )
+        .await
+        .unwrap();
+        writer
+            .put(vec![create_test_batch(&schema, 0, 3)])
+            .await
+            .unwrap();
+
+        let stats_handle = writer.stats_handle();
+        // `close()` drains the index-apply task, so the apply is settled here.
+        writer.close().await.unwrap();
+
+        let snapshot = stats_handle.snapshot();
+        assert!(
+            snapshot.index_update_count >= 1,
+            "the index apply must record an index-update stat, got {}",
+            snapshot.index_update_count
+        );
+        assert_eq!(
+            snapshot.index_update_rows, 3,
+            "every indexed row must be counted exactly once, got {}",
+            snapshot.index_update_rows
+        );
+        assert!(
+            snapshot.avg_index_update_latency().is_some(),
+            "a recorded index update must expose an average latency"
+        );
+    }
+
     #[tokio::test]
     async fn test_force_seal_active_and_wait_for_flush_drain() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
@@ -5353,9 +6877,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -5370,7 +6893,7 @@ mod tests {
             .manifest()
             .await
             .unwrap()
-            .map(|m| m.flushed_generations.len())
+            .map(|m| m.sstables.len())
             .unwrap_or(0);
 
         writer
@@ -5389,7 +6912,7 @@ mod tests {
             .await
             .unwrap()
             .expect("manifest should exist after flush");
-        assert_eq!(manifest.flushed_generations.len(), flushed_before + 1);
+        assert_eq!(manifest.sstables.len(), flushed_before + 1);
 
         writer.close().await.unwrap();
     }
@@ -5408,9 +6931,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -5428,7 +6950,7 @@ mod tests {
             .manifest()
             .await
             .unwrap()
-            .map(|m| m.flushed_generations.len())
+            .map(|m| m.sstables.len())
             .unwrap_or(0);
 
         writer.abort().await.unwrap();
@@ -5439,7 +6961,7 @@ mod tests {
             .manifest()
             .await
             .unwrap()
-            .map(|m| m.flushed_generations.len())
+            .map(|m| m.sstables.len())
             .unwrap_or(0);
         assert_eq!(
             flushed_after, flushed_before,
@@ -5463,9 +6985,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             // Short grace so the sweep is observable without a slow test.
@@ -5488,10 +7009,10 @@ mod tests {
         let manifest = writer.manifest().await.unwrap().expect("manifest exists");
         assert!(
             manifest
-                .flushed_generations
+                .sstables
                 .iter()
                 .any(|g| g.generation == initial_gen),
-            "flushed generation must be recorded in the manifest"
+            "SSTable must be recorded in the manifest"
         );
 
         // Still queryable in memory immediately after commit (within grace).
@@ -5499,7 +7020,7 @@ mod tests {
         assert_eq!(refs.active.generation, initial_gen + 1);
         assert!(
             refs.frozen.iter().any(|f| f.generation == initial_gen),
-            "flushed generation must stay queryable during the grace window"
+            "SSTable must stay queryable during the grace window"
         );
 
         // After the grace elapses (plus a sweep tick) the handle is evicted.
@@ -5523,9 +7044,8 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             frozen_memtable_grace: Duration::ZERO,
@@ -5547,10 +7067,10 @@ mod tests {
         let manifest = writer.manifest().await.unwrap().expect("manifest exists");
         assert!(
             manifest
-                .flushed_generations
+                .sstables
                 .iter()
                 .any(|g| g.generation == initial_gen),
-            "flushed generation must be recorded in the manifest"
+            "SSTable must be recorded in the manifest"
         );
 
         // ...and the in-memory handle is already gone, no sweep tick needed.
@@ -5563,6 +7083,54 @@ mod tests {
         writer.close().await.unwrap();
     }
 
+    #[tokio::test]
+    async fn test_close_propagates_frozen_memtable_flush_failure() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+        let writer_a = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+
+        let writer_b = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema,
+            vec![],
+        )
+        .await
+        .unwrap();
+        assert!(writer_b.epoch() > writer_a.epoch());
+
+        let error = writer_a
+            .close()
+            .await
+            .expect_err("close must propagate the fenced MemTable flush");
+        assert!(
+            matches!(error, Error::IO { .. }),
+            "unexpected error: {error}"
+        );
+        assert!(
+            error.to_string().contains("Writer fenced"),
+            "unexpected error: {error}"
+        );
+
+        writer_b.close().await.unwrap();
+    }
+
     /// Regression: a transient flush failure must NOT reopen the
     /// concurrent-read-vs-flush hole. The sealed generation stays in the
     /// queryable set (rows intact) until a later flush or WAL replay.
@@ -5786,9 +7354,7 @@ mod shard_writer_tests {
             Some("100")
         );
         // Every tunable field is present.
-        assert!(defaults.contains_key("sync_indexed_write"));
         assert!(defaults.contains_key("enable_memtable"));
-        assert!(defaults.contains_key("async_index_interval_ms"));
         // add_writer_config_default records arbitrary keys.
         assert_eq!(
             defaults.get("custom_knob").map(String::as_str),
@@ -5811,7 +7377,12 @@ mod shard_writer_tests {
 
         let vector_dim = 32;
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_multi_segment_index_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://multi-segment-index-{}/",
+            Uuid::new_v4().simple()
+        );
 
         // Initial fragment + an IVF vector index covering it.
         let initial = create_test_batch(&schema, 0, 256, vector_dim);
@@ -5949,7 +7520,7 @@ mod shard_writer_tests {
         // The tombstone-only generation still flushed (data without an HNSW index).
         let manifest = writer.manifest().await.unwrap().expect("manifest exists");
         assert_eq!(
-            manifest.flushed_generations.len(),
+            manifest.sstables.len(),
             1,
             "the all-tombstone generation must still flush"
         );
@@ -5995,9 +7566,7 @@ mod shard_writer_tests {
             .expect("Failed to initialize MemWAL");
 
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(true)
-            .with_sync_indexed_write(true);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(true);
         let writer = dataset
             .mem_wal_writer(shard_id, config)
             .await
@@ -6018,22 +7587,22 @@ mod shard_writer_tests {
             .await
             .expect("Failed to read manifest")
             .expect("Manifest should exist");
-        assert_eq!(manifest.flushed_generations.len(), 1);
+        assert_eq!(manifest.sstables.len(), 1);
 
-        let flushed = &manifest.flushed_generations[0];
-        let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, flushed.path);
-        let flushed_dataset = Dataset::open(&gen_uri)
+        let sstable = &manifest.sstables[0];
+        let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path);
+        let sstable = Dataset::open(&gen_uri)
             .await
-            .expect("Failed to open flushed generation");
-        let flushed_indices = flushed_dataset.load_indices().await.unwrap();
-        assert_eq!(flushed_indices.len(), 1);
-        assert_eq!(flushed_indices[0].name, "text_fts");
+            .expect("Failed to open SSTable");
+        let sstable_indices = sstable.load_indices().await.unwrap();
+        assert_eq!(sstable_indices.len(), 1);
+        assert_eq!(sstable_indices[0].name, "text_fts");
         assert_eq!(
-            flushed_indices[0].index_version, 1,
+            sstable_indices[0].index_version, 1,
             "maintained v1 FTS index must flush as v1"
         );
 
-        let results = flushed_dataset
+        let results = sstable
             .scan()
             .full_text_search(FullTextSearchQuery::new("Sample".to_owned()))
             .unwrap()
@@ -6049,7 +7618,12 @@ mod shard_writer_tests {
 
         let vector_dim = 32;
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_writer_hnsw_params_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://writer-hnsw-params-{}/",
+            Uuid::new_v4().simple()
+        );
 
         let initial = create_test_batch(&schema, 0, 256, vector_dim);
         let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
@@ -6330,9 +7904,7 @@ mod shard_writer_tests {
 
         // Create shard writer
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(false)
-            .with_sync_indexed_write(false);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(false);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -6357,7 +7929,12 @@ mod shard_writer_tests {
         let target_id = 1_000i64 + 37;
 
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_shard_writer_hnsw_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://shard-writer-hnsw-{}/",
+            Uuid::new_v4().simple()
+        );
 
         let initial_batch = create_test_batch(&schema, 0, 256, vector_dim);
         let batches = RecordBatchIterator::new([Ok(initial_batch)], schema.clone());
@@ -6385,9 +7962,7 @@ mod shard_writer_tests {
             .expect("Failed to initialize MemWAL");
 
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(true)
-            .with_sync_indexed_write(true);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(true);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -6526,9 +8101,7 @@ mod shard_writer_tests {
 
         // Create shard writer with default config
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(false)
-            .with_sync_indexed_write(false);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(false);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -6553,7 +8126,7 @@ mod shard_writer_tests {
     /// 2. File system layout is correct (WAL files, manifest, generation directories)
     /// 3. WAL entries contain expected data
     /// 4. Data can be read after each flush cycle
-    /// 5. Manifest tracks flushed generations correctly
+    /// 5. Manifest tracks SSTables correctly
     ///
     /// Run with: cargo test -p lance shard_writer_tests::test_shard_writer_e2e_correctness -- --nocapture
     #[tokio::test]
@@ -6605,7 +8178,6 @@ mod shard_writer_tests {
         let shard_id = Uuid::new_v4();
         let config = ShardWriterConfig::new(shard_id)
             .with_durable_write(true) // Ensure WAL files are written
-            .with_sync_indexed_write(true)
             .with_max_memtable_size(50 * 1024) // 50KB - triggers flush after ~8 batches
             .with_max_wal_buffer_size(10 * 1024) // 10KB WAL buffer
             .with_max_wal_flush_interval(Duration::from_millis(50)); // Fast flush
@@ -6682,24 +8254,24 @@ mod shard_writer_tests {
             .expect("Failed to read manifest")
             .expect("Manifest should exist");
 
-        // Verify flushed generations exist on disk
+        // Verify SSTables exist on disk
         assert!(
-            !manifest.flushed_generations.is_empty(),
-            "Should have at least one flushed generation"
+            !manifest.sstables.is_empty(),
+            "Should have at least one SSTable"
         );
-        for flushed_gen in &manifest.flushed_generations {
+        for sstable in &manifest.sstables {
             // The path stored in manifest is relative to the shard directory
             // Construct full path: temp_dir/_mem_wal/shard_id/generation_folder
             let gen_path = temp_dir
                 .path()
                 .join("_mem_wal")
                 .join(shard_id.to_string())
-                .join(&flushed_gen.path);
+                .join(&sstable.path);
 
             // The generation directory should exist
             assert!(
                 gen_path.exists(),
-                "Flushed generation directory should exist at {:?}",
+                "SSTable directory should exist at {:?}",
                 gen_path
             );
 
@@ -6729,9 +8301,13 @@ mod shard_writer_tests {
         // Re-open dataset and create new writer to verify recovery
         let dataset = Dataset::open(&uri).await.expect("Failed to reopen dataset");
         let new_shard_id = Uuid::new_v4();
-        let new_config = ShardWriterConfig::new(new_shard_id)
-            .with_durable_write(false)
-            .with_sync_indexed_write(true);
+        // `durable_write(true)` so the put waits for its flush, which is what
+        // currently publishes the rows. A non-durable put is *not* yet
+        // read-your-writes: the index apply is welded to the WAL flush, so the
+        // rows stay invisible until the next flush. This test used to pass with
+        // `durable_write(false)` only because an un-advanced cursor of 0 was
+        // misread as "batch 0 is visible" — it was asserting the dirty read.
+        let new_config = ShardWriterConfig::new(new_shard_id).with_durable_write(true);
 
         let new_writer = dataset
             .mem_wal_writer(new_shard_id, new_config)
@@ -6754,4 +8330,309 @@ mod shard_writer_tests {
             .await
             .expect("Failed to close new writer");
     }
+
+    /// Regression: a base opened with a *path-bound* store binding (the
+    /// deprecated `ObjectStoreParams::object_store`) must still flush and read
+    /// generations at their own paths.
+    ///
+    /// The binding pins a store to one location, and both
+    /// `ObjectStore::from_uri_and_params` and `DatasetBuilder::build_object_store`
+    /// take the path from it while ignoring the URI they were handed. Reusing the
+    /// base's params verbatim therefore aimed every generation write and open at
+    /// the base table itself: the flush failed ("dataset already exists") and any
+    /// derived open returned base rows as generation rows.
+    #[tokio::test]
+    async fn test_flush_and_read_with_path_bound_object_store() {
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use futures::TryStreamExt;
+        use lance_io::object_store::ObjectStoreParams;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        // Re-bind the base to a store pinned at the base's own path — what
+        // `DatasetBuilder::with_object_store` leaves on an opened dataset.
+        #[allow(deprecated)]
+        let store_params = ObjectStoreParams {
+            object_store: Some((
+                Arc::new(object_store::local::LocalFileSystem::new()),
+                url::Url::parse(&uri).unwrap(),
+            )),
+            ..Default::default()
+        };
+        let dataset = dataset.with_object_store(dataset.object_store.clone(), Some(store_params));
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer
+            .wait_for_flush_drain()
+            .await
+            .expect("flush must not be redirected at the base table");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert_eq!(manifest.sstables.len(), 1);
+        let sstable = manifest.sstables[0].clone();
+
+        // The generation landed under `_mem_wal/`, and the base table is untouched.
+        let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path);
+        let generation = Dataset::open(&gen_uri)
+            .await
+            .expect("generation must exist at its own path");
+        assert_eq!(generation.count_rows(None).await.unwrap(), 8);
+        let base = Dataset::open(&uri).await.unwrap();
+        assert_eq!(
+            base.count_rows(None).await.unwrap(),
+            16,
+            "the generation write must not land in the base table"
+        );
+
+        // The read path resolves the generation, not the base: 16 base + 8 flushed.
+        // Opening the base instead would dedup back down to 16 rows.
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_sstable(sstable.generation, sstable.path.clone());
+        let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]);
+        let rows: usize = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .expect("scan must open the generation, not the base")
+            .iter()
+            .map(|batch| batch.num_rows())
+            .sum();
+        assert_eq!(rows, 24);
+
+        writer.close().await.unwrap();
+    }
+
+    /// The store params a base was opened with must reach every *derived* open:
+    /// the flush that writes a generation and the scan that reads it back.
+    ///
+    /// This is the point of threading them at all. A namespace-vended store
+    /// exists only on the params (credentials, endpoint, wrapper), so a
+    /// generation resolved by URI alone would silently sign with the ambient
+    /// identity instead — succeeding against a local store and failing against
+    /// the vended one. Asserting on the *generation folder* rather than
+    /// `_mem_wal/` is what makes this bite: WAL entries are written through the
+    /// base dataset's own store, so they would show up here either way.
+    #[tokio::test]
+    async fn test_store_params_reach_generation_write_and_read() {
+        use crate::dataset::builder::DatasetBuilder;
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use crate::dataset::mem_wal::test_util::observable_store_params;
+        use futures::TryStreamExt;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+
+        // Open the base through an observable store, exactly as a namespace
+        // client would hand in a vended-credential store.
+        let (store_params, controls) = observable_store_params();
+        let mut dataset = DatasetBuilder::from_uri(&uri)
+            .with_store_params(store_params)
+            .load()
+            .await
+            .expect("Failed to open dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.expect("flush failed");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert_eq!(manifest.sstables.len(), 1);
+        let sstable = manifest.sstables[0].clone();
+
+        // The generation's own Lance manifest is the signal to key on. Keying on
+        // the generation folder alone would pass vacuously: sidecars like
+        // `{gen}/bloom_filter.bin` are written through the *base* dataset's
+        // store, which is observable no matter what the params do. And the
+        // fragments can't be used either — `ObjectStore::create` writes local
+        // files through `tokio::fs`, bypassing the object store entirely, so
+        // `{gen}/data/` never reaches a wrapper under `file://`. The manifest
+        // goes through `put_opts`, and only the flusher's `Dataset::write` /
+        // `open_generation` writes it — both of which must carry the params.
+        let gen_manifest = format!("{}/_versions", sstable.path);
+
+        assert!(
+            controls.wrote_under(&gen_manifest),
+            "the flush must write the generation through the base's store params, \
+             not a store resolved from the generation URI alone"
+        );
+
+        // And the read path must resolve the generation through them too.
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_sstable(sstable.generation, sstable.path.clone());
+        let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]);
+        let rows: usize = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .expect("scan failed")
+            .iter()
+            .map(|batch| batch.num_rows())
+            .sum();
+        assert_eq!(rows, 24);
+
+        // Reads key on the data files, not the manifest: the flusher already
+        // pulled the generation's manifest into the shared session cache, so the
+        // scan's open serves it from memory and never touches the store. The
+        // fragments are read through it (reads have no local bypass), as is the
+        // generation's standalone PK index.
+        assert!(
+            controls.read_under(&format!("{}/data/", sstable.path)),
+            "the scan must read the generation through the base's store params"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// A fresh-tier-only scanner reaches its store params through
+    /// `with_store_params`, not `new()`, so the setter must strip the path-bound
+    /// store binding too. Left raw, it redirects the generation open at the base
+    /// table and the scan silently returns base rows as WAL rows.
+    #[tokio::test]
+    async fn test_fresh_tier_scan_with_path_bound_object_store() {
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use futures::TryStreamExt;
+        use lance_io::object_store::ObjectStoreParams;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        // 16 base rows with ids 0..16; the WAL gets 8 rows with ids 1000..1008,
+        // so a redirected generation open is unambiguous in the output.
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.expect("flush failed");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        let sstable = manifest.sstables[0].clone();
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_sstable(sstable.generation, sstable.path.clone());
+
+        // What `DatasetBuilder::with_object_store` leaves on an opened dataset:
+        // a store pinned at the base's own path.
+        #[allow(deprecated)]
+        let store_params = ObjectStoreParams {
+            object_store: Some((
+                Arc::new(object_store::local::LocalFileSystem::new()),
+                url::Url::parse(&uri).unwrap(),
+            )),
+            ..Default::default()
+        };
+
+        let arrow_schema: Arc<ArrowSchema> = schema.clone();
+        let batches = LsmScanner::without_base_table(
+            arrow_schema,
+            uri.clone(),
+            vec![snapshot],
+            vec!["id".to_string()],
+        )
+        .with_session(dataset.session())
+        .with_store_params(store_params)
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .expect("scan must open the generation, not the base");
+
+        let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(
+            rows, 8,
+            "fresh tier holds only the 8 WAL rows; 16 means the generation open \
+             was redirected at the base table"
+        );
+        let ids: Vec<i64> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int64Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert!(
+            ids.iter().all(|id| (1_000..1_008).contains(id)),
+            "expected the WAL's own rows, got {ids:?}"
+        );
+
+        writer.close().await.unwrap();
+    }
 }
diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs
index 656f0163ce6..a4c9a845fe0 100644
--- a/rust/lance/src/dataset/optimize.rs
+++ b/rust/lance/src/dataset/optimize.rs
@@ -88,7 +88,7 @@ use std::ops::{AddAssign, Range};
 use std::sync::Arc;
 
 use super::fragment::FileFragment;
-use super::index::DatasetIndexRemapperOptions;
+use super::index::{DatasetIndexRemapperOptions, load_indices_for_remapping};
 use super::rowids::load_row_id_sequences;
 use super::transaction::{
     Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder,
@@ -114,7 +114,7 @@ use lance_core::Error;
 use lance_core::datatypes::{BlobHandling, BlobKind};
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::{DATASET_COMPACTING_EVENT, TRACE_DATASET_EVENTS};
-use lance_index::frag_reuse::FragReuseGroup;
+use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseGroup};
 use lance_index::is_system_index;
 use lance_table::format::{Fragment, RowIdMeta};
 use roaring::{RoaringBitmap, RoaringTreemap};
@@ -262,6 +262,15 @@ pub struct CompactionOptions {
     /// fragments at a time).
     /// Defaults to `None` (no limit, all eligible fragments are compacted).
     pub max_source_fragments: Option<usize>,
+    /// Maximum number of data overlay files a fragment may carry before it is
+    /// fully compacted. When set, any fragment with more than this many overlays
+    /// is rewritten into a fresh fragment with its overlays (and deletions)
+    /// materialized into the base data, dropping the fragment from any index
+    /// left stale by those overlays.
+    /// Defaults to `Some(10)`. Set to `Some(0)` to compact every fragment that
+    /// carries any overlay, or `None` to disable the overlay-count trigger
+    /// entirely.
+    pub max_overlays_per_fragment: Option<usize>,
     /// Transaction properties to store with this commit.
     ///
     /// These key-value pairs are stored in the transaction file
@@ -291,6 +300,7 @@ impl Default for CompactionOptions {
             enable_binary_copy_force: false,
             binary_copy_read_batch_bytes: Some(16 * 1024 * 1024),
             max_source_fragments: None,
+            max_overlays_per_fragment: Some(10),
             transaction_properties: None,
         }
     }
@@ -317,6 +327,7 @@ impl CompactionOptions {
     /// - `lance.compaction.compaction_mode`
     /// - `lance.compaction.binary_copy_read_batch_bytes`
     /// - `lance.compaction.max_source_fragments`
+    /// - `lance.compaction.max_overlays_per_fragment`
     pub fn from_dataset_config(config: &HashMap<String, String>) -> Result<Self> {
         let mut opts = Self::default();
         opts.apply_dataset_config(config)?;
@@ -427,6 +438,19 @@ impl CompactionOptions {
                         ))
                     })?);
                 }
+                "max_overlays_per_fragment" => {
+                    // The default is `Some(10)`, so an explicit "none" is the only
+                    // way to disable the trigger through the manifest config.
+                    self.max_overlays_per_fragment = match value.to_ascii_lowercase().as_str() {
+                        "none" => None,
+                        _ => Some(value.parse().map_err(|_| {
+                            Error::invalid_input(format!(
+                                "Invalid value for {}: '{}' (expected a non-negative integer or 'none')",
+                                key, value
+                            ))
+                        })?),
+                    };
+                }
                 _ => {
                     warn!("Ignoring unknown compaction config key: {}", key);
                 }
@@ -714,7 +738,16 @@ impl CompactionPlanner for DefaultCompactionPlanner {
         while let Some(res) = fragment_metrics.next().await {
             let (fragment, metrics) = res?;
 
-            let candidacy = if self.options.materialize_deletions
+            let over_overlay_limit = self
+                .options
+                .max_overlays_per_fragment
+                .is_some_and(|max| fragment.overlays.len() > max);
+
+            let candidacy = if over_overlay_limit {
+                // Too many overlays: fully compact this fragment on its own,
+                // regardless of its size or deletion count.
+                Some(CompactionCandidacy::CompactItself)
+            } else if self.options.materialize_deletions
                 && metrics.deletion_percentage() > self.options.materialize_deletions_threshold
             {
                 Some(CompactionCandidacy::CompactItself)
@@ -840,7 +873,7 @@ pub async fn compact_files_with_planner(
 
     let dataset_ref = &dataset.clone();
 
-    let result_stream = futures::stream::iter(compaction_plan.tasks.into_iter())
+    let result_stream = futures::stream::iter(compaction_plan.tasks)
         .map(|task| rewrite_files(Cow::Borrowed(dataset_ref), task, &compaction_plan.options))
         .buffer_unordered(
             compaction_plan
@@ -938,7 +971,7 @@ impl CompactionPlan {
 
 /// Classification for one blob v2 row during compaction.
 ///
-/// - `Null`: NULL row or Inline blob with position=0 and size=0.
+/// - `Null`: NULL row.
 /// - `External`: External blob referenced by URI.
 /// - `DataBlob`: Inline/Packed/Dedicated blob stored in Lance files.
 enum RowClass {
@@ -947,25 +980,6 @@ enum RowClass {
     DataBlob,
 }
 
-/// Check if a row is a null Inline blob.
-///
-/// This matches `BlobV2StructuralEncoder`'s behavior of encoding null rows as
-/// Inline with position=0 and size=0, and `collect_blob_entries_v2`'s behavior
-/// of skipping them.
-fn is_inline_null_blob(
-    kind: BlobKind,
-    position_col: &arrow::array::UInt64Array,
-    size_col: &arrow::array::UInt64Array,
-    index: usize,
-) -> bool {
-    if kind != BlobKind::Inline {
-        return false;
-    }
-    let position_is_empty = position_col.is_null(index) || position_col.value(index) == 0;
-    let size_is_empty = size_col.is_null(index) || size_col.value(index) == 0;
-    position_is_empty && size_is_empty
-}
-
 /// Column views for the 5 fields in a blob v2 descriptor struct.
 struct BlobV2Descriptor<'a> {
     kind_col: &'a arrow::array::UInt8Array,
@@ -1062,8 +1076,6 @@ fn classify_rows(
             })?;
             if kind == BlobKind::External {
                 row_classes.push(RowClass::External);
-            } else if is_inline_null_blob(kind, descriptor.position_col, descriptor.size_col, i) {
-                row_classes.push(RowClass::Null);
             } else {
                 row_classes.push(RowClass::DataBlob);
                 blob_read_addrs.push(row_addrs.value(i));
@@ -1139,7 +1151,13 @@ async fn build_user_view_struct(
                 }
             }
             RowClass::DataBlob => {
-                let data = blob_files[blob_file_idx].read().await?;
+                let blob_file = blob_files[blob_file_idx].as_ref().ok_or_else(|| {
+                    Error::internal(format!(
+                        "Non-null blob row {} in column '{}' resolved to null",
+                        i, column_name
+                    ))
+                })?;
+                let data = blob_file.read().await?;
                 blob_file_idx += 1;
                 data_builder.append_value(data.as_ref());
                 uri_builder.append_null();
@@ -1161,10 +1179,11 @@ async fn build_user_view_struct(
     )?)
 }
 
-async fn transform_blob_v2_batch(
+pub(crate) async fn transform_blob_v2_batch(
     dataset: &Arc<Dataset>,
     schema: &lance_core::datatypes::Schema,
     batch: RecordBatch,
+    keep_row_addr: bool,
 ) -> Result<RecordBatch> {
     let row_addr_idx = batch
         .schema()
@@ -1188,7 +1207,7 @@ async fn transform_blob_v2_batch(
 
     let batch_schema = batch.schema();
     for (col_idx, field) in batch_schema.fields().iter().enumerate() {
-        if field.name() == lance_core::ROW_ADDR {
+        if field.name() == lance_core::ROW_ADDR && !keep_row_addr {
             continue;
         }
 
@@ -1213,6 +1232,15 @@ async fn transform_blob_v2_batch(
                 ))
             })?;
 
+        // Merge-insert may supply a blob v2 value directly from the source.
+        // Those values are already in the writer's user view and do not refer
+        // to a row in the target dataset, unlike descriptor values from scans.
+        if struct_arr.column_by_name("kind").is_none() {
+            new_columns.push(batch.column(col_idx).clone());
+            new_fields.push(field.clone());
+            continue;
+        }
+
         let column_name = field.name();
         let descriptor = BlobV2Descriptor::try_from_struct(struct_arr, column_name)?;
         let classification = classify_rows(struct_arr, &descriptor, row_addrs, column_name)?;
@@ -1463,7 +1491,12 @@ async fn load_index_fragmaps(dataset: &Dataset) -> Result<Vec<RoaringBitmap>> {
             index_fragmaps.push(fragment_bitmap.clone());
         } else {
             let dataset_at_index = dataset.checkout_version(index.dataset_version).await?;
-            let frags = 0..dataset_at_index.manifest.max_fragment_id.unwrap_or(0);
+            // max_fragment_id is inclusive (the highest id); +1 for an exclusive
+            // upper bound so the last fragment is covered (None => empty range).
+            let frags = 0..dataset_at_index
+                .manifest
+                .max_fragment_id
+                .map_or(0, |m| m + 1);
             index_fragmaps.push(RoaringBitmap::from_sorted_iter(frags).unwrap());
         }
     }
@@ -1570,8 +1603,13 @@ async fn rewrite_files(
         .iter()
         .map(|f| f.physical_rows.unwrap() as u64)
         .sum::<u64>();
-    // If we aren't using stable row ids, then we need to remap indices.
-    let needs_remapping = !dataset.manifest.uses_stable_row_ids();
+    // Capturing row addresses is only useful if something will consume them:
+    // an index to remap now, or a deferred remap through the FRI.
+    let capture_row_addrs = !dataset.manifest.uses_stable_row_ids()
+        && (options.defer_index_remap
+            || load_indices_for_remapping(dataset.as_ref())
+                .await?
+                .is_some());
     let mut new_fragments: Vec<Fragment>;
     let task_id = uuid::Uuid::new_v4();
     log::info!(
@@ -1597,7 +1635,7 @@ async fn rewrite_files(
             options.batch_size,
             options.io_buffer_size,
             true,
-            needs_remapping,
+            capture_row_addrs,
         )
         .await?;
         row_ids_rx = rx_initial;
@@ -1622,7 +1660,7 @@ async fn rewrite_files(
                 let schema = dataset_schema.clone();
                 async move {
                     let batch = batch_result?;
-                    transform_blob_v2_batch(&dataset, &schema, batch)
+                    transform_blob_v2_batch(&dataset, &schema, batch, false)
                         .await
                         .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))
                 }
@@ -1701,7 +1739,7 @@ async fn rewrite_files(
             ));
         }
 
-        if needs_remapping {
+        if capture_row_addrs {
             let (tx, rx) = std::sync::mpsc::channel();
             let mut addrs = RoaringTreemap::new();
             for frag in &fragments {
@@ -1941,8 +1979,8 @@ async fn recalc_versions_for_rewritten_fragments(
     // Set both version metadata on new fragments
     for ((fragment, last_updated_seq), created_at_seq) in new_fragments
         .iter_mut()
-        .zip(new_last_updated_sequences.into_iter())
-        .zip(new_created_at_sequences.into_iter())
+        .zip(new_last_updated_sequences)
+        .zip(new_created_at_sequences)
     {
         fragment.last_updated_at_version_meta = Some(
             lance_table::format::RowDatasetVersionMeta::from_sequence(&last_updated_seq).unwrap(),
@@ -1971,8 +2009,17 @@ pub async fn commit_compaction(
         return Ok(CompactionMetrics::default());
     }
 
-    // If we aren't using stable row ids, then we need to remap indices.
-    let needs_remapping = !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap;
+    let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some());
+    // Address-style results require immediate index remapping unless it is deferred.
+    let needs_remapping =
+        !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap && has_address_style;
+
+    // Confirm there is a remapper before materializing the potentially very large row address map.
+    let index_remapper = if needs_remapping {
+        remap_options.create_remapper(dataset).await?
+    } else {
+        None
+    };
 
     // Determine the earliest version at which compaction tasks were planned/executed.
     //
@@ -1996,7 +2043,6 @@ pub async fn commit_compaction(
     let mut completed_tasks = completed_tasks;
 
     // Single reserve_fragment_ids for all address-style tasks
-    let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some());
     if has_address_style {
         let frags: Vec<&mut Fragment> = completed_tasks
             .iter_mut()
@@ -2014,6 +2060,31 @@ pub async fn commit_compaction(
     let mut frag_reuse_groups: Vec<FragReuseGroup> = Vec::new();
     let mut new_fragment_bitmap: RoaringBitmap = RoaringBitmap::new();
 
+    // Write an FRI only when the compaction touches data an index must later
+    // remap: a rewrite group covered by a data index, or by the existing FRI's new
+    // fragments (the composed remap chain). Compacting only not-yet-indexed data
+    // needs no FRI (one written for it is un-drainable). Decide all-or-nothing per
+    // compaction, never per group -- a partial FRI is unsound: a concurrent reindex
+    // can make a skipped fragment indexed and the conflict resolver's FRI-present
+    // path won't re-check it.
+    let indexed_frags: RoaringBitmap = if options.defer_index_remap {
+        let mut covered = RoaringBitmap::new();
+        for bm in load_index_fragmaps(dataset).await? {
+            covered |= bm;
+        }
+        if let Some(bm) = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await?
+            .and_then(|fri| fri.fragment_bitmap)
+        {
+            covered |= bm;
+        }
+        covered
+    } else {
+        RoaringBitmap::new()
+    };
+    let mut any_group_indexed = false;
+
     for task in completed_tasks {
         metrics += task.metrics;
         let rewrite_group = RewriteGroup {
@@ -2021,7 +2092,7 @@ pub async fn commit_compaction(
             new_fragments: task.new_fragments.clone(),
         };
 
-        if needs_remapping {
+        if index_remapper.is_some() {
             if let Some(row_addrs_bytes) = task.row_addrs {
                 let row_addrs =
                     RoaringTreemap::deserialize_from(&mut Cursor::new(&row_addrs_bytes))?;
@@ -2062,6 +2133,14 @@ pub async fn commit_compaction(
                 }
             }
         } else if options.defer_index_remap {
+            // Record every group; track whether any touches indexed/chain data.
+            if task
+                .original_fragments
+                .iter()
+                .any(|f| indexed_frags.contains(f.id as u32))
+            {
+                any_group_indexed = true;
+            }
             let changed_row_addrs = task.row_addrs.ok_or_else(|| {
                 Error::internal(
                     "defer_index_remap requires row_addrs but none were provided".to_string(),
@@ -2080,8 +2159,7 @@ pub async fn commit_compaction(
         rewrite_groups.push(rewrite_group);
     }
 
-    let rewritten_indices = if needs_remapping {
-        let index_remapper = remap_options.create_remapper(dataset)?;
+    let rewritten_indices = if let Some(index_remapper) = index_remapper {
         let affected_ids = rewrite_groups
             .iter()
             .flat_map(|group| group.old_fragments.iter().map(|frag| frag.id))
@@ -2116,9 +2194,15 @@ pub async fn commit_compaction(
         Vec::new()
     };
 
-    let frag_reuse_index = if options.defer_index_remap {
+    // No indexed/chain data touched -> no FRI (all-or-nothing, see above).
+    let frag_reuse_index = if options.defer_index_remap && any_group_indexed {
         Some(build_new_frag_reuse_index(dataset, frag_reuse_groups, new_fragment_bitmap).await?)
     } else {
+        if options.defer_index_remap {
+            log::debug!(
+                "skipping fragment-reuse index: no rewritten fragments were covered by an index"
+            );
+        }
         None
     };
 
@@ -2192,6 +2276,7 @@ mod tests {
     use lance_datagen::Dimension;
     use lance_file::version::LanceFileVersion;
     use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
+    use lance_index::frag_reuse::FragReuseIndexHandle;
     use lance_index::scalar::{
         BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
     };
@@ -2221,6 +2306,7 @@ mod tests {
         let fragment = Fragment {
             id: 0,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(0),
@@ -2273,6 +2359,20 @@ mod tests {
         .unwrap()
     }
 
+    /// Build (or, with `replace`, rebuild) a scalar index named "scalar" on `col`.
+    async fn create_scalar_index(dataset: &mut Dataset, col: &str, replace: bool) {
+        dataset
+            .create_index(
+                &[col],
+                IndexType::Scalar,
+                Some("scalar".into()),
+                &ScalarIndexParams::default(),
+                replace,
+            )
+            .await
+            .unwrap();
+    }
+
     #[derive(Debug, Default, Clone, PartialEq)]
     struct MockIndexRemapperExpectation {
         expected: HashMap<u64, Option<u64>>,
@@ -2355,9 +2455,10 @@ mod tests {
         }
     }
 
+    #[async_trait]
     impl IndexRemapperOptions for MockIndexRemapper {
-        fn create_remapper(&self, _: &Dataset) -> Result<Box<dyn IndexRemapper>> {
-            Ok(Box::new(self.clone()))
+        async fn create_remapper(&self, _: &Dataset) -> Result<Option<Box<dyn IndexRemapper>>> {
+            Ok(Some(Box::new(self.clone())))
         }
     }
 
@@ -2493,7 +2594,7 @@ mod tests {
             .unwrap();
         assert_eq!(blobs.len(), expected_payload.len());
         for (blob, expected) in blobs.iter().zip(expected_payload.iter()) {
-            let bytes = blob.read().await.unwrap();
+            let bytes = blob.as_ref().unwrap().read().await.unwrap();
             assert_eq!(bytes.as_ref(), expected.as_slice());
         }
     }
@@ -2886,9 +2987,70 @@ mod tests {
         }
     }
 
+    #[async_trait]
     impl IndexRemapperOptions for IgnoreRemap {
-        fn create_remapper(&self, _: &Dataset) -> Result<Box<dyn IndexRemapper>> {
-            Ok(Box::new(Self {}))
+        async fn create_remapper(&self, _: &Dataset) -> Result<Option<Box<dyn IndexRemapper>>> {
+            Ok(None)
+        }
+    }
+
+    #[rstest]
+    #[case::without_index(false)]
+    #[case::with_index(true)]
+    #[tokio::test]
+    async fn test_row_addrs_only_used_with_remappable_index(#[case] has_index: bool) {
+        let data = sample_data();
+        let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 9_000))], data.schema());
+        let mut dataset = Dataset::write(
+            reader,
+            "memory://",
+            Some(WriteParams {
+                max_rows_per_file: 3_000,
+                data_storage_version: Some(LanceFileVersion::Legacy),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        if has_index {
+            create_scalar_index(&mut dataset, "a", false).await;
+        }
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 9_000,
+            ..Default::default()
+        };
+        let plan = plan_compaction(&dataset, &options).await.unwrap();
+        assert_eq!(plan.tasks().len(), 1);
+
+        let mut result = rewrite_files(Cow::Borrowed(&dataset), plan.tasks()[0].clone(), &options)
+            .await
+            .unwrap();
+        assert_eq!(result.row_addrs.is_some(), has_index);
+
+        if has_index {
+            let row_addrs_bytes = result
+                .row_addrs
+                .as_ref()
+                .expect("indexed compaction should capture row addresses");
+            let row_addrs =
+                RoaringTreemap::deserialize_from(&mut Cursor::new(row_addrs_bytes)).unwrap();
+            assert_eq!(row_addrs.len(), 9_000);
+        } else {
+            // Simulate a stale worker result that captured row addresses before the
+            // dataset no longer needed a remapper. Invalid bytes ensure the commit
+            // does not attempt to deserialize or materialize the unused map.
+            result.row_addrs = Some(b"not a roaring treemap".to_vec());
+            commit_compaction(
+                &mut dataset,
+                vec![result],
+                Arc::new(DatasetIndexRemapperOptions::default()),
+                &options,
+            )
+            .await
+            .unwrap();
+            assert_eq!(dataset.get_fragments().len(), 1);
         }
     }
 
@@ -3114,6 +3276,10 @@ mod tests {
 
         assert_eq!(dataset.get_fragments().len(), num_fragments);
 
+        // An FRI is only written for compactions that touch indexed data, so
+        // index the column being compacted.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         // Delete a few rows from each fragment so compaction has something to do.
         dataset.delete("i % 1000 = 0").await.unwrap();
 
@@ -3225,17 +3391,10 @@ mod tests {
         dataset.delete("i < 500").await.unwrap();
         dataset2.delete("i < 500").await.unwrap();
 
-        // Create a scalar index to check this is not touched
-        dataset
-            .create_index(
-                &["i"],
-                IndexType::Scalar,
-                Some("scalar".into()),
-                &ScalarIndexParams::default(),
-                false,
-            )
-            .await
-            .unwrap();
+        // Create the same scalar index on both datasets so deferred and immediate
+        // remapping are compared under the same conditions.
+        create_scalar_index(&mut dataset, "i", false).await;
+        create_scalar_index(&mut dataset2, "i", false).await;
 
         // Verify the initial state - no fragment reuse index should exist
         let initial_indices = dataset.load_indices().await.unwrap();
@@ -3369,7 +3528,9 @@ mod tests {
             open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref())
                 .await
                 .unwrap();
-        let stats = frag_reuse_index.statistics().unwrap();
+        let stats = FragReuseIndexHandle(Arc::new(frag_reuse_index.clone()))
+            .statistics()
+            .unwrap();
         assert_eq!(
             serde_json::to_string(&stats).unwrap(),
             dataset
@@ -3427,6 +3588,52 @@ mod tests {
         assert_eq!(current_scalar_index.uuid, original_scalar_uuid);
     }
 
+    #[tokio::test]
+    async fn test_defer_index_remap_skips_fri_when_no_indexed_data() {
+        // A deferred compaction touching no indexed data must write no FRI --
+        // such a version is un-drainable (remap no-ops, trim retains it forever).
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned())));
+
+        let mut dataset = Dataset::write(
+            data_gen.batch(600),
+            "memory://test/noindex",
+            Some(WriteParams {
+                max_rows_per_file: 100, // 6 small files -> compaction has work
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // No index at all: nothing covers any fragment.
+        assert!(dataset.load_indices().await.unwrap().is_empty());
+        let fragments_before = dataset.get_fragments().len();
+        assert!(fragments_before > 1, "need multiple fragments to compact");
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 100_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+        compact_files(&mut dataset, options, None).await.unwrap();
+
+        // Compaction actually ran...
+        assert!(
+            dataset.get_fragments().len() < fragments_before,
+            "compaction should have merged fragments"
+        );
+        // ...but no fragment-reuse index was created.
+        assert!(
+            dataset
+                .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+                .await
+                .unwrap()
+                .is_none(),
+            "deferred compaction with no indexed data must not create an FRI"
+        );
+    }
+
     #[tokio::test]
     async fn test_defer_index_remap_multiple_compactions() {
         let mut data_gen = BatchGenerator::new()
@@ -3446,6 +3653,10 @@ mod tests {
         .await
         .unwrap();
 
+        // FRI is written only for compactions touching indexed data; index "i" so
+        // the successive deferred compactions build a chained fragment-reuse index.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -3498,13 +3709,100 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_defer_index_remap_mixed_records_all_groups() {
+        // All-or-nothing: a compaction touching any indexed data records the full
+        // FRI, including the unindexed group (a per-group filter would drop it).
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned())));
+        let mut dataset = Dataset::write(
+            data_gen.batch(300),
+            "memory://test/mixed",
+            Some(WriteParams {
+                max_rows_per_file: 100, // 3 fragments
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Index the initial fragments, then append more that stay unindexed.
+        create_scalar_index(&mut dataset, "i", false).await;
+        Dataset::write(
+            data_gen.batch(300),
+            WriteDestination::Dataset(Arc::new(dataset.clone())),
+            Some(WriteParams {
+                max_rows_per_file: 100, // 3 more, unindexed
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        dataset.checkout_latest().await.unwrap();
+
+        // Fragments not covered by the scalar index are the "unindexed" ones.
+        let indexed: HashSet<u32> = dataset
+            .load_index_by_name("scalar")
+            .await
+            .unwrap()
+            .unwrap()
+            .fragment_bitmap
+            .unwrap()
+            .iter()
+            .collect();
+        let unindexed_frags: Vec<u64> = dataset
+            .fragments()
+            .iter()
+            .map(|f| f.id)
+            .filter(|id| !indexed.contains(&(*id as u32)))
+            .collect();
+        assert!(
+            !unindexed_frags.is_empty(),
+            "expected some unindexed fragments"
+        );
+
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 100_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        // All-or-nothing: because indexed fragments were compacted, the FRI is
+        // written AND records the unindexed group too (a per-group filter would
+        // have dropped it).
+        let fri_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("mixed compaction must write an FRI");
+        let details = load_frag_reuse_index_details(&dataset, &fri_meta)
+            .await
+            .unwrap();
+        let recorded_old: HashSet<u64> = details
+            .versions
+            .iter()
+            .flat_map(|v| v.old_frag_ids())
+            .collect();
+        for f in &unindexed_frags {
+            assert!(
+                recorded_old.contains(f),
+                "unindexed fragment {f} must be recorded in the FRI (all-or-nothing)"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_deferred_compaction_not_split_by_frag_reuse_index() {
-        // A deferred compaction creates a fragment-reuse index covering its
-        // output. Later small fragments must still compact together with that
-        // (FRI-covered) output: the FRI is a system index and must not split the
-        // compaction bin. Without the fix the FRI-covered fragment is isolated,
-        // so only the new fragments merge and the count never returns to one.
+        // The fragment-reuse index is a system index and must be excluded from
+        // compaction bin planning; otherwise its covered fragment is isolated and
+        // the small fragments never coalesce back to one.
         let data = sample_data();
         let test_dir = TempStrDir::default();
         let test_uri = &test_dir;
@@ -3526,6 +3824,11 @@ mod tests {
         )
         .await
         .unwrap();
+
+        // Index "a" so the deferred compaction records an FRI (only written for
+        // compactions touching indexed data). The FRI is a system index and must
+        // still not split later compaction bins -- the property this test guards.
+        create_scalar_index(&mut dataset, "a", false).await;
         compact_files(&mut dataset, options.clone(), None)
             .await
             .unwrap();
@@ -3553,11 +3856,16 @@ mod tests {
         .unwrap();
         assert_eq!(dataset.get_fragments().len(), 3);
 
+        // Reindex so every fragment is data-indexed -- then the FRI (a system
+        // index, correctly excluded from bin planning) is the only thing that
+        // could split the bin.
+        create_scalar_index(&mut dataset, "a", true).await;
+
         compact_files(&mut dataset, options, None).await.unwrap();
         assert_eq!(
             dataset.get_fragments().len(),
             1,
-            "FRI-covered fragment must compact together with the new fragments"
+            "FRI (a system index) must not split the compaction bin; all fragments coalesce"
         );
     }
 
@@ -3881,6 +4189,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -3977,9 +4288,13 @@ mod tests {
             .unwrap();
         let new_frags3 = frag_reuse_details3.versions.last().unwrap().new_frag_ids();
 
-        // Concurrently commit a frag_reuse_index cleanup operation.
-        // Because there is no index, it should remove the first version.
-        // but after rebase it should contain the new compaction versions.
+        // Concurrently commit a frag_reuse_index cleanup operation. dataset_clone
+        // only knows the first reuse version; catch its index up so the cleanup
+        // removes that version. After rebase onto the other compactions it should
+        // contain the new compaction versions.
+        remapping::remap_column_index(&mut dataset_clone, &["i"], Some("scalar".into()))
+            .await
+            .unwrap();
         cleanup_frag_reuse_index(&mut dataset_clone).await.unwrap();
 
         // Load and verify the fragment reuse index content
@@ -4019,6 +4334,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -4057,8 +4375,12 @@ mod tests {
             .unwrap();
         assert_eq!(frag_reuse_details.versions.len(), 1);
 
-        // First commit the frag_reuse_index cleanup
-        // Because there is no index, it should remove the first version.
+        // Catch the index up to the compaction (on `dataset` only; `dataset_clone`
+        // keeps the un-caught-up index for the concurrent rewrite below), then
+        // clean up: with the index caught up the trim removes the first version.
+        remapping::remap_column_index(&mut dataset, &["i"], Some("scalar".into()))
+            .await
+            .unwrap();
         cleanup_frag_reuse_index(&mut dataset).await.unwrap();
 
         // Load and verify the fragment reuse index content
@@ -4129,6 +4451,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -6264,6 +6589,29 @@ mod tests {
         assert!(err_msg.contains("invalid_mode"));
     }
 
+    #[test]
+    fn test_from_dataset_config_max_overlays_per_fragment() {
+        let key = "lance.compaction.max_overlays_per_fragment".to_string();
+
+        // An integer sets the threshold.
+        let config = HashMap::from([(key.clone(), "3".to_string())]);
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.max_overlays_per_fragment, Some(3));
+
+        // "none" (case-insensitive) disables the trigger, overriding the Some(10) default.
+        let config = HashMap::from([(key.clone(), "None".to_string())]);
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.max_overlays_per_fragment, None);
+
+        // Anything else is rejected.
+        let config = HashMap::from([(key, "not_a_number".to_string())]);
+        let err_msg = CompactionOptions::from_dataset_config(&config)
+            .unwrap_err()
+            .to_string();
+        assert!(err_msg.contains("max_overlays_per_fragment"));
+        assert!(err_msg.contains("not_a_number"));
+    }
+
     #[test]
     fn test_apply_dataset_config_overrides() {
         let config = HashMap::from([(
@@ -6885,16 +7233,130 @@ mod tests {
             let row_id = row_ids.value(i);
             let id = ids.value(i);
             let blobs = dataset.take_blobs(&[row_id], column).await.unwrap();
-            if blobs.is_empty() {
-                result.push((id, None));
-            } else {
-                let data = blobs[0].read().await.unwrap();
-                result.push((id, Some(data.to_vec())));
+            match blobs.into_iter().next().flatten() {
+                Some(blob) => {
+                    let data = blob.read().await.unwrap();
+                    result.push((id, Some(data.to_vec())));
+                }
+                None => result.push((id, None)),
             }
         }
         result
     }
 
+    fn mixed_blob_values() -> Vec<(i32, Option<Vec<u8>>)> {
+        vec![
+            (0, Some(vec![b'0'; 80])),
+            (1, None),
+            (2, Some(Vec::new())),
+            (3, Some(vec![b'3'; 80])),
+            (4, Some(vec![b'4'; 80])),
+            (5, Some(vec![b'5'; 80])),
+        ]
+    }
+
+    async fn assert_compaction_preserves_blob_values(
+        mut dataset: Dataset,
+        expected: &[(i32, Option<Vec<u8>>)],
+    ) {
+        assert_eq!(dataset.get_fragments().len(), 3);
+
+        let mut before = read_blob_bytes_by_index(&Arc::new(dataset.clone()), "blob").await;
+        before.sort_by_key(|(id, _)| *id);
+        assert_eq!(before, expected);
+
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 1024 * 1024,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(dataset.get_fragments().len(), 1);
+
+        let mut after = read_blob_bytes_by_index(&Arc::new(dataset), "blob").await;
+        after.sort_by_key(|(id, _)| *id);
+        assert_eq!(after, expected);
+    }
+
+    #[tokio::test]
+    async fn test_compact_blob_v1_preserves_null_empty_and_payload_order() {
+        let test_dir = TempStrDir::default();
+        let expected = mixed_blob_values();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("blob", DataType::LargeBinary, true)
+                .with_metadata([(BLOB_META_KEY.to_string(), "true".to_string())].into()),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)),
+                Arc::new(LargeBinaryArray::from_iter(
+                    expected.iter().map(|(_, value)| value.as_deref()),
+                )),
+            ],
+        )
+        .unwrap();
+        let dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch)], schema),
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_0),
+                max_rows_per_file: 2,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_compaction_preserves_blob_values(dataset, &expected).await;
+    }
+
+    #[tokio::test]
+    async fn test_compact_blob_v2_preserves_null_empty_and_payload_order() {
+        use crate::BlobArrayBuilder;
+
+        let test_dir = TempStrDir::default();
+        let expected = mixed_blob_values();
+        let mut blob_builder = BlobArrayBuilder::new(expected.len());
+        for (_, value) in &expected {
+            match value {
+                Some(value) => blob_builder.push_bytes(value).unwrap(),
+                None => blob_builder.push_null().unwrap(),
+            }
+        }
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            crate::blob_field("blob", true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)),
+                blob_builder.finish().unwrap(),
+            ],
+        )
+        .unwrap();
+        let dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch)], schema),
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                max_rows_per_file: 2,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_compaction_preserves_blob_values(dataset, &expected).await;
+    }
+
     #[tokio::test]
     async fn test_compact_blob_v2_preserves_external_references() {
         use crate::BlobArrayBuilder;
@@ -7885,4 +8347,339 @@ mod tests {
             ]
         );
     }
+    // ---- `max_overlays_per_fragment` compaction trigger ----
+    //
+    // Tests for the trigger that fully compacts a fragment carrying too many data
+    // overlay files into a fresh fragment with the overlays (and deletions)
+    // materialized into the base data.
+    use arrow_array::record_batch;
+    use lance_file::writer::{FileWriter, FileWriterOptions};
+    use lance_io::utils::CachedFileSize;
+    use lance_table::format::DataFile;
+    use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+    use std::collections::BTreeMap;
+
+    use crate::dataset::DATA_DIR;
+    use crate::dataset::transaction::DataOverlayGroup;
+
+    /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) =
+    /// id * 10, six rows per fragment (fragments 0 and 1).
+    async fn create_base_dataset(uri: &str) -> Dataset {
+        let batch = record_batch!(
+            ("id", Int32, (0..12).collect::<Vec<_>>()),
+            ("val", Int32, (0..12).map(|v| v * 10).collect::<Vec<_>>())
+        )
+        .unwrap();
+        let schema = batch.schema();
+        let write_params = WriteParams {
+            max_rows_per_file: 6,
+            max_rows_per_group: 6,
+            data_storage_version: Some(LanceFileVersion::Stable),
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        Dataset::write(reader, uri, Some(write_params))
+            .await
+            .unwrap()
+    }
+
+    fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+        Arc::new(Int32Array::from_iter(values))
+    }
+
+    fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+        RoaringBitmap::from_iter(offsets)
+    }
+
+    /// Write a dense overlay covering `fields` of `fragment_id` with `columns`
+    /// as the per-field value columns, then commit it as a `DataOverlay`.
+    async fn commit_overlay(
+        dataset: Dataset,
+        fragment_id: u64,
+        fields: &[i32],
+        coverage: OverlayCoverage,
+        columns: Vec<ArrayRef>,
+    ) -> Dataset {
+        let read_version = dataset.version().version;
+        let overlay_schema = dataset.schema().project_by_ids(fields, true);
+        let filename = format!("{}.lance", Uuid::new_v4());
+        let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str());
+        let obj_writer = dataset.object_store.create(&path).await.unwrap();
+        let mut writer = FileWriter::try_new(
+            obj_writer,
+            overlay_schema,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        let (major, minor) = writer.version().to_numbers();
+        for (column_index, array) in columns.into_iter().enumerate() {
+            writer.write_column(column_index, array).await.unwrap();
+        }
+        let summary = writer.finish().await.unwrap();
+
+        let mut data_file = DataFile::new_unstarted(filename, major, minor);
+        data_file.fields = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(f, _)| *f as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.column_indices = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(_, c)| *c as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+        Dataset::commit(
+            WriteDestination::Dataset(Arc::new(dataset)),
+            Operation::DataOverlay {
+                groups: vec![DataOverlayGroup {
+                    fragment_id,
+                    overlays: vec![DataOverlayFile {
+                        data_file,
+                        coverage,
+                        committed_version: 0,
+                    }],
+                }],
+            },
+            Some(read_version),
+            None,
+            None,
+            Arc::new(Default::default()),
+            false,
+        )
+        .await
+        .unwrap()
+    }
+
+    /// Commit `n` distinct single-cell overlays to fragment 0 (offset `i`, val
+    /// column set to `1000 + i`), so the fragment ends up with `n` overlays. The
+    /// `1000 +` offset keeps overlaid values clear of the base `id * 10` values.
+    async fn commit_n_overlays(mut dataset: Dataset, n: u32) -> Dataset {
+        for i in 0..n {
+            dataset = commit_overlay(
+                dataset,
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([i])),
+                vec![i32_array([Some(1000 + i as i32)])],
+            )
+            .await;
+        }
+        dataset
+    }
+
+    /// Options whose only compaction trigger is the overlay limit: base
+    /// fragments here are far below the default 1M-row target, which would
+    /// otherwise make them size-based compaction candidates on their own.
+    fn overlay_only_options(max_overlays_per_fragment: usize) -> CompactionOptions {
+        CompactionOptions {
+            max_overlays_per_fragment: Some(max_overlays_per_fragment),
+            target_rows_per_fragment: 6,
+            ..Default::default()
+        }
+    }
+
+    /// Scan `id` and `val` and return an `id -> val` map (order-independent).
+    async fn id_val_map(dataset: &Dataset) -> BTreeMap<i32, Option<i32>> {
+        let mut scanner = dataset.scan();
+        scanner.project(&["id", "val"]).unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let mut out = BTreeMap::new();
+        let ids = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        let vals = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        for i in 0..batch.num_rows() {
+            let v = if vals.is_null(i) {
+                None
+            } else {
+                Some(vals.value(i))
+            };
+            out.insert(ids.value(i), v);
+        }
+        out
+    }
+
+    #[tokio::test]
+    async fn test_max_overlays_triggers_full_compaction() {
+        // Fragment 0 gets 3 overlays; fragment 1 stays clean.
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+        assert_eq!(
+            dataset.get_fragment(0).unwrap().metadata().overlays.len(),
+            3
+        );
+
+        // Threshold 2: only fragment 0 (3 > 2) is compacted.
+        let metrics = compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+        assert_eq!(metrics.fragments_removed, 1);
+        assert_eq!(metrics.fragments_added, 1);
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+        // The compacted fragment is a fresh single-data-file fragment with no
+        // overlays; fragment 1 is untouched.
+        let compacted = fragments
+            .iter()
+            .find(|f| f.id() != 1)
+            .expect("a new fragment id was assigned");
+        assert!(compacted.metadata().overlays.is_empty());
+        assert_eq!(compacted.metadata().files.len(), 1);
+
+        // The overlaid values were materialized: id i in 0..3 -> 1000 + i.
+        let values = id_val_map(&dataset).await;
+        let expected: BTreeMap<i32, Option<i32>> = (0..12)
+            .map(|id| {
+                let v = if id < 3 { 1000 + id } else { id * 10 };
+                (id, Some(v))
+            })
+            .collect();
+        assert_eq!(values, expected);
+    }
+
+    #[tokio::test]
+    async fn test_below_threshold_is_a_noop() {
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 2).await;
+
+        // 2 overlays, threshold 2: `overlays > max` is false, so no compaction.
+        let metrics = compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+        assert_eq!(metrics.fragments_removed, 0);
+        assert_eq!(metrics.fragments_added, 0);
+        assert_eq!(
+            dataset.get_fragment(0).unwrap().metadata().overlays.len(),
+            2
+        );
+    }
+
+    #[tokio::test]
+    async fn test_overlay_compaction_materializes_deletions() {
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+        // Delete a row from the overlaid fragment (id 2 is at offset 2).
+        dataset.delete("id = 2").await.unwrap();
+        assert!(
+            dataset
+                .get_fragment(0)
+                .unwrap()
+                .metadata()
+                .deletion_file
+                .is_some()
+        );
+
+        compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+
+        // The deletion was materialized: no deletion file remains and id 2 is gone.
+        for fragment in dataset.get_fragments() {
+            assert!(fragment.metadata().deletion_file.is_none());
+            assert!(fragment.metadata().overlays.is_empty());
+        }
+        let values = id_val_map(&dataset).await;
+        assert!(!values.contains_key(&2));
+        // Surviving overlaid cells still carry their materialized values.
+        assert_eq!(values.get(&0), Some(&Some(1000)));
+        assert_eq!(values.get(&1), Some(&Some(1001)));
+    }
+
+    #[tokio::test]
+    async fn test_overlay_compaction_reconciles_stale_index() {
+        let mut dataset = create_base_dataset("memory://").await;
+        // Index `val` before any overlay -> the index is stale once val is overlaid.
+        dataset
+            .create_index(
+                &["val"],
+                IndexType::Scalar,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Overlay val[0] 0 -> 100 (committed after the index) and push fragment 0
+        // over the overlay limit.
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+
+        let val_index_before = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .find(|i| i.fields == vec![1])
+            .expect("val index present")
+            .clone();
+        assert!(
+            val_index_before
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .contains(0)
+        );
+
+        compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+
+        // The stale val index no longer covers the compacted fragment, so its
+        // rows fall back to a flat scan instead of serving stale values.
+        let indices = dataset.load_indices().await.unwrap();
+        let val_index = indices
+            .iter()
+            .find(|i| i.fields == vec![1])
+            .expect("val index present");
+        let compacted_id = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .find(|id| *id != 1)
+            .unwrap();
+        assert!(
+            !val_index
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .contains(compacted_id),
+            "stale index must drop the compacted fragment from its coverage"
+        );
+
+        // The indexed query is correct: the materialized value is found and the
+        // stale pre-overlay value is gone.
+        let mut scanner = dataset.scan();
+        scanner
+            .filter("val = 1000")
+            .unwrap()
+            .project(&["id"])
+            .unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let ids = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(ids.len(), 1);
+        assert_eq!(ids.value(0), 0);
+
+        let mut scanner = dataset.scan();
+        scanner.filter("val = 0").unwrap().project(&["id"]).unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        assert_eq!(batch.num_rows(), 0, "stale value 0 must no longer match");
+    }
 }
diff --git a/rust/lance/src/dataset/optimize/remapping.rs b/rust/lance/src/dataset/optimize/remapping.rs
index aef2cd231fc..682ed696f21 100644
--- a/rust/lance/src/dataset/optimize/remapping.rs
+++ b/rust/lance/src/dataset/optimize/remapping.rs
@@ -61,8 +61,13 @@ pub trait IndexRemapper: Send + Sync {
 ///
 /// Currently we don't have any options but we may need options in the future and so we
 /// want to keep a placeholder
+#[async_trait]
 pub trait IndexRemapperOptions: Send + Sync {
-    fn create_remapper(&self, dataset: &Dataset) -> Result<Box<dyn IndexRemapper>>;
+    /// Creates a remapper when the dataset has indices that need row address remapping.
+    ///
+    /// Returns `None` when no remappable indices exist, allowing compaction to avoid
+    /// materializing an unused row address map.
+    async fn create_remapper(&self, dataset: &Dataset) -> Result<Option<Box<dyn IndexRemapper>>>;
 }
 
 #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
@@ -75,9 +80,10 @@ impl IndexRemapper for IgnoreRemap {
     }
 }
 
+#[async_trait]
 impl IndexRemapperOptions for IgnoreRemap {
-    fn create_remapper(&self, _: &Dataset) -> Result<Box<dyn IndexRemapper>> {
-        Ok(Box::new(Self {}))
+    async fn create_remapper(&self, _: &Dataset) -> Result<Option<Box<dyn IndexRemapper>>> {
+        Ok(None)
     }
 }
 
diff --git a/rust/lance/src/dataset/overlay.rs b/rust/lance/src/dataset/overlay.rs
new file mode 100644
index 00000000000..83d4bc17938
--- /dev/null
+++ b/rust/lance/src/dataset/overlay.rs
@@ -0,0 +1,1427 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Resolution of data overlay files on read.
+//!
+//! An overlay supplies replacement values for some `(row, field)` cells without
+//! rewriting the base data. Resolving a read means, for each row we return,
+//! deciding whether its value comes from the base column or from an overlay.
+//!
+//! Three coordinate spaces show up throughout this module; keeping them straight
+//! is most of the work:
+//!
+//! - `offset_in_frag`: a row's physical position in the fragment (0-based over all
+//!   physical rows, ignoring deletions). This is how a cell is addressed on disk
+//!   and in an overlay's coverage bitmap.
+//! - `offset_in_batch`: a row's position within the batch we are currently
+//!   assembling (0-based). The output column is indexed by this.
+//! - `offset_in_overlay`: the position of a value in an overlay's value column.
+//!   An overlay stores its values densely — one per covered cell, in ascending
+//!   `offset_in_frag` order — so a covered cell's value is found by counting how
+//!   many covered cells come before it. (That count is what a roaring bitmap calls
+//!   the cell's "rank".)
+//!
+//! For a given field, the overlays covering it are consulted newest to oldest: the
+//! first overlay that covers a row wins, and its value is read at that row's
+//! `offset_in_overlay`. A row that no overlay covers keeps its base value.
+//!
+//! The rows to resolve are passed in as a list of `offset_in_frag` (one per output
+//! row), so a single code path serves both scans (a contiguous range of offsets)
+//! and `take` (arbitrary offsets).
+//!
+//! Deletions win over overlays, but nothing here handles that: the merge runs on
+//! physical rows *before* deletions are applied, so an overlay value computed for a
+//! deleted row is simply dropped along with the row. This matches the spec with no
+//! special casing.
+
+use std::collections::{BTreeSet, HashMap};
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef, RecordBatch, StructArray};
+use arrow_select::interleave::interleave;
+use futures::StreamExt;
+use lance_core::datatypes::{Field, Schema};
+use lance_core::{Error, Result};
+use roaring::RoaringBitmap;
+
+use lance_table::format::overlay::DataOverlayFile;
+use lance_table::format::{DataFile, Fragment, IndexMetadata};
+use lance_table::utils::stream::ReadBatchFut;
+
+use crate::dataset::fragment::{FileFragment, FragReadConfig, GenericFileReader};
+
+/// The physical offsets within a fragment whose value for an indexed field may be
+/// stale relative to an index built at `index_version`, and so must be excluded
+/// from that index's results and re-evaluated against current values on the flat
+/// path.
+///
+/// The set is the union, over every overlay whose `committed_version` is newer
+/// than `index_version`, of that overlay's coverage **restricted to the indexed
+/// fields**. The restriction makes exclusion field-aware: an overlay that touches
+/// only non-indexed fields contributes nothing. An overlay whose
+/// `committed_version <= index_version` is already incorporated by the index and
+/// is ignored.
+pub fn overlay_exclusion_offsets(
+    overlays: &[DataOverlayFile],
+    indexed_field_ids: &[i32],
+    index_version: u64,
+    schema: &Schema,
+) -> Result<RoaringBitmap> {
+    let mut excluded = RoaringBitmap::new();
+    for overlay in overlays {
+        if overlay.committed_version <= index_version {
+            continue;
+        }
+        for (field_pos, field_id) in overlay.data_file.fields.iter().enumerate() {
+            let overlay_ancestry = schema.field_ancestry_by_id(*field_id);
+            let affects_index = indexed_field_ids.iter().any(|indexed_field_id| {
+                indexed_field_id == field_id
+                    || overlay_ancestry.as_ref().is_some_and(|ancestry| {
+                        ancestry
+                            .iter()
+                            .any(|ancestor| ancestor.id == *indexed_field_id)
+                    })
+                    || schema
+                        .field_ancestry_by_id(*indexed_field_id)
+                        .is_some_and(|ancestry| {
+                            ancestry.iter().any(|ancestor| ancestor.id == *field_id)
+                        })
+            });
+            if affects_index {
+                excluded |= &*overlay.coverage_for_field(field_pos)?;
+            }
+        }
+    }
+    Ok(excluded)
+}
+
+// Stale row offsets contributed by one fragment's overlays for a given index version.
+// Applies a cheap version gate first: if every overlay predates the segment it is already
+// incorporated by the index, so there is nothing stale and the field/bitmap work is skipped.
+fn stale_offsets_for_fragment(
+    fragment: &Fragment,
+    fields: &[i32],
+    index_version: u64,
+    schema: &Schema,
+) -> Result<RoaringBitmap> {
+    if fragment
+        .overlays
+        .iter()
+        .all(|o| o.committed_version <= index_version)
+    {
+        return Ok(RoaringBitmap::new());
+    }
+    overlay_exclusion_offsets(&fragment.overlays, fields, index_version, schema)
+}
+
+// A missing `fragment_bitmap` means the index predates fragment-bitmap tracking; treat it as
+// covering every fragment (matching `DatasetPreFilter::new`) so overlay-stale rows can't slip
+// through unmasked. Only skip fragments explicitly absent from a present bitmap.
+fn covers_fragment(coverage: Option<&RoaringBitmap>, frag_id: u32) -> bool {
+    coverage.is_none_or(|c| c.contains(frag_id))
+}
+
+/// Index by fragment id the fragments that carry at least one overlay. Overlays are rare, so
+/// this is empty on the common path, letting callers skip index loading entirely; when non-empty
+/// it bounds the stale-collection loops to `O(overlaid fragments)`.
+pub fn overlaid_fragments(fragments: &[Fragment]) -> HashMap<u32, &Fragment> {
+    fragments
+        .iter()
+        .filter(|f| !f.overlays.is_empty())
+        .map(|f| (f.id as u32, f))
+        .collect()
+}
+
+/// Insert into `stale` the ids of fragments covered by `segment` whose index entries may be
+/// stale because an overlay committed after the segment was built touches a field the segment
+/// indexes. Field-aware and version-gated via [`overlay_exclusion_offsets`].
+///
+/// `overlaid_frags` holds only the fragments that actually carry overlays (rare), so the loop is
+/// `O(overlaid_frags)` rather than `O(fragments the segment covers)`.
+pub fn collect_overlay_stale_frags(
+    segment: &IndexMetadata,
+    overlaid_frags: &HashMap<u32, &Fragment>,
+    stale: &mut RoaringBitmap,
+    schema: &Schema,
+) -> Result<()> {
+    let coverage = segment.fragment_bitmap.as_ref();
+    for (&frag_id, fragment) in overlaid_frags {
+        if stale.contains(frag_id) || !covers_fragment(coverage, frag_id) {
+            continue;
+        }
+        if !stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)?
+            .is_empty()
+        {
+            stale.insert(frag_id);
+        }
+    }
+    Ok(())
+}
+
+/// Like [`collect_overlay_stale_frags`] but with row-level granularity: instead of marking the
+/// whole fragment stale, it computes exactly which row offsets within each covered fragment are
+/// stale and accumulates them into `stale` (fragment_id → stale row offsets).
+///
+/// Used by the scalar and vector paths to block only the affected rows from index results and
+/// re-evaluate only those rows on the flat path, keeping overhead proportional to the number of
+/// overlaid rows rather than the whole fragment size.
+pub fn collect_overlay_stale_rows_for_segment(
+    segment: &IndexMetadata,
+    overlaid_frags: &HashMap<u32, &Fragment>,
+    stale: &mut HashMap<u32, RoaringBitmap>,
+    schema: &Schema,
+) -> Result<()> {
+    let coverage = segment.fragment_bitmap.as_ref();
+    for (&frag_id, fragment) in overlaid_frags {
+        if !covers_fragment(coverage, frag_id) {
+            continue;
+        }
+        let excluded =
+            stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)?;
+        if !excluded.is_empty() {
+            *stale.entry(frag_id).or_default() |= &excluded;
+        }
+    }
+    Ok(())
+}
+
+/// The plan for merging one field's overlays into one batch: which source (base or
+/// a particular overlay) supplies each output row, and which overlay values must be
+/// fetched to do it.
+///
+/// Built by [`route_overlays`] from the coverage bitmaps alone — before any value
+/// column is read — so the caller can fetch only the overlay values it will
+/// actually use (its `offsets_in_overlay`) rather than whole columns, then build the
+/// merged column with [`assemble_overlay_column`].
+struct OverlayRouting {
+    /// One `(source, position)` pair per output row, ready to hand to `interleave`.
+    /// Source `0` is the base column, with `position` = the row's `offset_in_batch`;
+    /// source `k + 1` is overlay `k`'s fetched values, with `position` = the row's
+    /// index into those fetched values.
+    indices: Vec<(usize, usize)>,
+    /// Per overlay (newest-first): the sorted, deduplicated `offset_in_overlay`
+    /// values this batch needs from that overlay — i.e. exactly which entries of its
+    /// value column to fetch.
+    offsets_in_overlay: Vec<Vec<u32>>,
+    /// Whether any row is covered by an overlay at all (false ⇒ every row falls
+    /// through to the base column, so the base is already the answer and no overlay
+    /// values need to be read).
+    any_overlay: bool,
+}
+
+/// For each row in `offsets_in_frag`, decide whether its value comes from the base
+/// column or from an overlay — and if from an overlay, at which `offset_in_overlay`.
+///
+/// Only the coverage bitmaps are consulted (newest-first), so this runs before any
+/// value column is read and reports exactly which overlay values the caller must
+/// fetch.
+///
+/// A scan asks for a contiguous, ascending range of offsets, which enables a faster
+/// bitmap-driven path ([`route_contiguous`]); `take` asks for arbitrary offsets and
+/// uses the general path ([`route_arbitrary`]). Both produce identical routing.
+fn route_overlays(
+    offsets_in_frag: &[u32],
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    match contiguous_frag_start(offsets_in_frag) {
+        Some(frag_start) => {
+            route_contiguous(frag_start, offsets_in_frag.len(), coverages_newest_first)
+        }
+        None => route_arbitrary(offsets_in_frag, coverages_newest_first),
+    }
+}
+
+/// If `offsets_in_frag` is a contiguous ascending run `[start, start + 1, ...]`,
+/// return `start`; otherwise `None` (including when empty).
+fn contiguous_frag_start(offsets_in_frag: &[u32]) -> Option<u32> {
+    let start = *offsets_in_frag.first()?;
+    offsets_in_frag
+        .iter()
+        .enumerate()
+        .all(|(i, &offset)| offset as u64 == start as u64 + i as u64)
+        .then_some(start)
+}
+
+/// Fast path for a scan, where the batch is a contiguous run of offsets starting at
+/// `frag_start`. Because the offsets are contiguous, a row's `offset_in_batch` is
+/// just `offset_in_frag - frag_start`, so a coverage's set bits map straight to
+/// output rows — no need to test each row against each coverage.
+///
+/// For each coverage we intersect it with the batch's offset range. Roaring does
+/// this a block at a time, so a coverage that does not overlap the batch (e.g. a
+/// scan batch past the last cell this overlay touches) is skipped cheaply without
+/// inspecting individual bits.
+///
+/// Within the batch a coverage's cells appear in ascending order, so their
+/// `offset_in_overlay` values are consecutive: the first in-batch cell sits at
+/// `offset_in_overlay = <cells this coverage has before the batch>` (a single
+/// `rank` lookup), and each following cell is one more. Coverages are applied
+/// newest-first, and the first overlay to claim a row wins.
+fn route_contiguous(
+    frag_start: u32,
+    len: usize,
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    let mut offsets_in_overlay: Vec<Vec<u32>> = vec![Vec::new(); coverages_newest_first.len()];
+    // Indexed by offset_in_batch: which (overlay, fetch position) supplies the row.
+    let mut routed: Vec<Option<(usize, usize)>> = vec![None; len];
+    let range_end = (frag_start as u64 + len as u64).min(u32::MAX as u64) as u32;
+    let mut batch_range = RoaringBitmap::new();
+    batch_range.insert_range(frag_start..range_end);
+
+    for (k, coverage) in coverages_newest_first.iter().enumerate() {
+        let covered_in_batch = *coverage & &batch_range;
+        if covered_in_batch.is_empty() {
+            continue;
+        }
+        // offset_in_overlay of this coverage's first in-batch cell = the number of
+        // its cells that lie before the batch.
+        let first_offset_in_overlay = if frag_start == 0 {
+            0
+        } else {
+            coverage.rank(frag_start - 1) as u32
+        };
+        for (nth_in_batch, offset_in_frag) in covered_in_batch.iter().enumerate() {
+            let offset_in_batch = (offset_in_frag - frag_start) as usize;
+            if routed[offset_in_batch].is_none() {
+                routed[offset_in_batch] = Some((k, offsets_in_overlay[k].len()));
+                offsets_in_overlay[k].push(first_offset_in_overlay + nth_in_batch as u32);
+            }
+        }
+    }
+
+    let mut any_overlay = false;
+    let indices = routed
+        .into_iter()
+        .enumerate()
+        .map(|(offset_in_batch, routed)| match routed {
+            None => (0, offset_in_batch),
+            Some((k, fetch_pos)) => {
+                any_overlay = true;
+                (k + 1, fetch_pos)
+            }
+        })
+        .collect();
+
+    OverlayRouting {
+        indices,
+        offsets_in_overlay,
+        any_overlay,
+    }
+}
+
+/// General path for arbitrary offsets (e.g. `take`): test each row's
+/// `offset_in_frag` against the coverages newest-first. `take` batches are small,
+/// so this `O(rows * overlays)` probing is not a concern.
+fn route_arbitrary(
+    offsets_in_frag: &[u32],
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    // Per overlay: the distinct offset_in_overlay values this batch needs, sorted.
+    let mut offset_sets: Vec<BTreeSet<u32>> = vec![BTreeSet::new(); coverages_newest_first.len()];
+    // Per output row: the (overlay, offset_in_overlay) that supplies it, if any.
+    let mut routed_per_row: Vec<Option<(usize, u32)>> = Vec::with_capacity(offsets_in_frag.len());
+    for &offset_in_frag in offsets_in_frag {
+        let mut routed = None;
+        for (k, coverage) in coverages_newest_first.iter().enumerate() {
+            if coverage.contains(offset_in_frag) {
+                // offset_in_overlay = number of covered cells before this one.
+                let offset_in_overlay = coverage.rank(offset_in_frag) as u32 - 1;
+                offset_sets[k].insert(offset_in_overlay);
+                routed = Some((k, offset_in_overlay));
+                break;
+            }
+        }
+        routed_per_row.push(routed);
+    }
+
+    let offsets_in_overlay: Vec<Vec<u32>> = offset_sets
+        .iter()
+        .map(|offsets| offsets.iter().copied().collect())
+        .collect();
+    // For each overlay, map an offset_in_overlay to its position in the fetched
+    // (sorted, deduplicated) value list.
+    let fetch_positions: Vec<HashMap<u32, usize>> = offsets_in_overlay
+        .iter()
+        .map(|offsets| {
+            offsets
+                .iter()
+                .enumerate()
+                .map(|(pos, &o)| (o, pos))
+                .collect()
+        })
+        .collect();
+
+    let mut any_overlay = false;
+    let indices = routed_per_row
+        .into_iter()
+        .enumerate()
+        .map(|(offset_in_batch, routed)| match routed {
+            None => (0, offset_in_batch),
+            Some((k, offset_in_overlay)) => {
+                any_overlay = true;
+                (k + 1, fetch_positions[k][&offset_in_overlay])
+            }
+        })
+        .collect();
+
+    OverlayRouting {
+        indices,
+        offsets_in_overlay,
+        any_overlay,
+    }
+}
+
+/// Build the merged column from `base` and the overlay values fetched for the
+/// `offset_in_overlay` values [`route_overlays`] asked for.
+///
+/// `fetched_newest_first[k]` holds overlay `k`'s values for `routing`'s
+/// `offsets_in_overlay[k]`, in that order. The result has the same length and
+/// type as `base`. A covered row whose overlay value is NULL resolves **to** NULL
+/// (distinct from a fall-through, which keeps the base value).
+fn assemble_overlay_column(
+    base: &ArrayRef,
+    routing: &OverlayRouting,
+    fetched_newest_first: &[ArrayRef],
+) -> Result<ArrayRef> {
+    if !routing.any_overlay {
+        return Ok(base.clone());
+    }
+    if fetched_newest_first.len() != routing.offsets_in_overlay.len() {
+        return Err(Error::invalid_input(format!(
+            "overlay assembly got {} value columns but routing expects {}",
+            fetched_newest_first.len(),
+            routing.offsets_in_overlay.len()
+        )));
+    }
+    for (k, values) in fetched_newest_first.iter().enumerate() {
+        if values.len() != routing.offsets_in_overlay[k].len() {
+            return Err(Error::invalid_input(format!(
+                "overlay value column {} has {} values but {} were requested",
+                k,
+                values.len(),
+                routing.offsets_in_overlay[k].len()
+            )));
+        }
+    }
+
+    let mut sources: Vec<&dyn Array> = Vec::with_capacity(fetched_newest_first.len() + 1);
+    sources.push(base.as_ref());
+    for values in fetched_newest_first {
+        sources.push(values.as_ref());
+    }
+    interleave(&sources, &routing.indices).map_err(Error::from)
+}
+
+/// One overlay's contribution to one projected atomic field, with its file reader opened.
+#[derive(Debug, Clone)]
+struct LoadedAtomicFieldOverlay {
+    /// The `offset_in_frag` cells this overlay covers for the atomic field.
+    coverage: Arc<RoaringBitmap>,
+    /// Reader over the overlay data file, projected to the covered atomic fields; shared
+    /// across the atomic fields that the same file covers.
+    reader: Arc<dyn GenericFileReader>,
+}
+
+/// The overlays that apply to a single projected atomic field — a per-row field an overlay
+/// can replace as a unit (a primitive leaf, or a whole list/map field; structs are
+/// recursed through, not treated as atomic fields). Ordered newest-first, with readers opened
+/// and pruned to a specific read. Produced by [`resolve_overlays`] and consumed by
+/// [`merge_overlay_batch`].
+#[derive(Debug, Clone)]
+pub struct LoadedAtomicField {
+    /// The top-level output column the atomic field lives in (its name locates the batch
+    /// column; its field tree drives the descend/splice into that column).
+    top_field: Arc<Field>,
+    /// Child field ids from `top_field` down to the atomic field (empty when the atomic
+    /// field *is* the top-level column). Drives descending to, and splicing back, the
+    /// atomic field.
+    ancestor_ids: Vec<i32>,
+    /// Projection of exactly the atomic field (its ancestor path pruned to the atomic
+    /// field subtree), used to fetch the atomic field's values from the overlay file.
+    fetch_projection: Arc<Schema>,
+    overlays_newest_first: Vec<LoadedAtomicFieldOverlay>,
+}
+
+/// One overlay file that may contribute to a read, before it is opened. Opened
+/// lazily by [`resolve_overlays`], and only if the read actually touches it.
+#[derive(Debug, Clone)]
+struct PlannedOverlayFile {
+    data_file: DataFile,
+    /// The covered ∩ projected atomic fields to project when the file is opened, so a single
+    /// reader serves every atomic field the file contributes to.
+    open_projection: Arc<Schema>,
+}
+
+/// One overlay's contribution to one projected atomic field, before the file is opened.
+#[derive(Debug, Clone)]
+struct PlannedAtomicFieldOverlay {
+    /// Index into [`OverlayReadPlanner::files`] of the file that supplies the value.
+    file: usize,
+    coverage: Arc<RoaringBitmap>,
+}
+
+/// The overlays that apply to a single projected atomic field, ordered newest-first, before
+/// any file is opened.
+#[derive(Debug, Clone)]
+struct PlannedAtomicField {
+    top_field: Arc<Field>,
+    ancestor_ids: Vec<i32>,
+    fetch_projection: Arc<Schema>,
+    overlays_newest_first: Vec<PlannedAtomicFieldOverlay>,
+}
+
+/// A fragment's overlay-resolution plan for a projection, derived from coverage
+/// metadata alone — no file opened, no IO. [`resolve_overlays`] turns it into opened
+/// [`LoadedAtomicField`]s for one specific read, opening only the files whose cells
+/// the read's rows actually touch.
+#[derive(Debug, Clone)]
+pub struct OverlayReadPlanner {
+    files: Vec<PlannedOverlayFile>,
+    atomic_fields: Vec<PlannedAtomicField>,
+}
+
+impl OverlayReadPlanner {
+    /// True when no projected atomic field has any overlay, so there is nothing to resolve.
+    pub fn is_empty(&self) -> bool {
+        self.atomic_fields.is_empty()
+    }
+}
+
+/// Plan `fragment`'s overlay resolution for a projection from coverage metadata
+/// alone. No files are opened here (see [`resolve_overlays`]) — this only reads the
+/// already-parsed coverage bitmaps, so it is cheap enough to run on every open.
+///
+/// Overlays are stored oldest-first (sorted newest-last on load, see
+/// `sort_overlays_newest_last`), so walking them in reverse gives newest-first
+/// precedence.
+///
+/// Resolution is per *atomic field* — a per-row field that an overlay replaces as a unit: a
+/// primitive leaf, or a whole list/map field. Structs are internal nodes, so each
+/// leaf of a struct is its own atomic field and can be overlaid independently of its
+/// siblings. An overlay is written against the leaf ids it stores (the V2_1
+/// structural encoding records only leaves), so an overlay contributes to a projected
+/// atomic field when any id in its `data_file.fields` falls in that atomic field's leaf
+/// set. At merge time the atomic field's value is fetched and spliced into its output
+/// column, so an overlay on a sub-field never disturbs the column's other leaves. Each
+/// contributing overlay *file* appears once in `files`, shared by every atomic field it
+/// covers.
+pub fn plan_overlays(fragment: &FileFragment, projection: &Schema) -> Result<OverlayReadPlanner> {
+    let overlays = &fragment.metadata.overlays;
+    debug_assert!(
+        overlays
+            .windows(2)
+            .all(|w| w[0].committed_version <= w[1].committed_version),
+        "overlays must be sorted newest-last (see sort_overlays_newest_last)"
+    );
+
+    // The projection's atomic fields, and a leaf-id -> atomic-field-index map so an
+    // overlay's stored leaf ids resolve to the atomic field they belong to in O(1).
+    struct AtomicFieldInfo<'a> {
+        top_field: &'a Field,
+        ancestor_ids: Vec<i32>,
+        atomic_field_id: i32,
+    }
+    let mut atomic_field_infos: Vec<AtomicFieldInfo> = Vec::new();
+    let mut leaf_to_atomic_field: HashMap<i32, usize> = HashMap::new();
+    for top in &projection.fields {
+        for (atomic_field, ancestor_ids) in enumerate_atomic_fields(top) {
+            let idx = atomic_field_infos.len();
+            let mut value_leaf_ids = Vec::new();
+            collect_leaf_ids(atomic_field, &mut value_leaf_ids);
+            for leaf in value_leaf_ids {
+                leaf_to_atomic_field.insert(leaf, idx);
+            }
+            atomic_field_infos.push(AtomicFieldInfo {
+                top_field: top,
+                ancestor_ids,
+                atomic_field_id: atomic_field.id,
+            });
+        }
+    }
+
+    // Walk overlays newest-first. For each overlay, find the atomic fields it covers and push
+    // (newest-first, for free) into their per-atomic field overlay lists.
+    let mut files = Vec::new();
+    let mut atomic_field_overlays: Vec<Vec<PlannedAtomicFieldOverlay>> =
+        vec![Vec::new(); atomic_field_infos.len()];
+    for overlay in overlays.iter().rev() {
+        // atomic field index -> the `data_file.fields` position whose coverage to read. An
+        // overlay writes one value per row per atomic field, so its leaves share a coverage;
+        // the first leaf of each atomic field to appear wins.
+        let mut covered: HashMap<usize, usize> = HashMap::new();
+        for (field_pos, &field_id) in overlay.data_file.fields.iter().enumerate() {
+            if let Some(&atomic_field_idx) = leaf_to_atomic_field.get(&field_id) {
+                covered.entry(atomic_field_idx).or_insert(field_pos);
+            }
+        }
+        if covered.is_empty() {
+            continue;
+        }
+        let file = files.len();
+        let covered_ids: Vec<i32> = covered
+            .keys()
+            .map(|&i| atomic_field_infos[i].atomic_field_id)
+            .collect();
+        files.push(PlannedOverlayFile {
+            data_file: overlay.data_file.clone(),
+            open_projection: Arc::new(projection.project_by_ids(&covered_ids, true)),
+        });
+        for (atomic_field_idx, field_pos) in covered {
+            atomic_field_overlays[atomic_field_idx].push(PlannedAtomicFieldOverlay {
+                file,
+                coverage: overlay.coverage_for_field(field_pos)?,
+            });
+        }
+    }
+
+    // Emit one PlannedAtomicField per projected atomic field that has overlays, in
+    // atomic field order.
+    let mut atomic_fields = Vec::new();
+    for (idx, info) in atomic_field_infos.iter().enumerate() {
+        let overlays_newest_first = std::mem::take(&mut atomic_field_overlays[idx]);
+        if overlays_newest_first.is_empty() {
+            continue;
+        }
+        atomic_fields.push(PlannedAtomicField {
+            top_field: Arc::new(info.top_field.clone()),
+            ancestor_ids: info.ancestor_ids.clone(),
+            fetch_projection: Arc::new(projection.project_by_ids(&[info.atomic_field_id], true)),
+            overlays_newest_first,
+        });
+    }
+    Ok(OverlayReadPlanner {
+        files,
+        atomic_fields,
+    })
+}
+
+/// The per-row atomic fields of a projected top-level field, each with the child-id path from
+/// the top-level field down to it. Structs are recursed through; a primitive leaf or a
+/// whole list/map field is an atomic field (values are one-per-row). A top-level primitive or
+/// list yields a single atomic field with an empty path.
+fn enumerate_atomic_fields(top: &Field) -> Vec<(&Field, Vec<i32>)> {
+    fn recurse<'a>(field: &'a Field, path: &mut Vec<i32>, out: &mut Vec<(&'a Field, Vec<i32>)>) {
+        if field.logical_type.is_struct() {
+            for child in &field.children {
+                path.push(child.id);
+                recurse(child, path, out);
+                path.pop();
+            }
+        } else {
+            out.push((field, path.clone()));
+        }
+    }
+    let mut out = Vec::new();
+    let mut path = Vec::new();
+    recurse(top, &mut path, &mut out);
+    out
+}
+
+/// Collect the leaf field ids in `field`'s subtree — the ids an overlay stores for
+/// this atomic field (its own id if primitive; its item leaves if a list/map).
+fn collect_leaf_ids(field: &Field, out: &mut Vec<i32>) {
+    if field.children.is_empty() {
+        out.push(field.id);
+    } else {
+        for child in &field.children {
+            collect_leaf_ids(child, out);
+        }
+    }
+}
+
+/// Follow a path of child field ids from `field` down through nested structs, taking
+/// the corresponding child array at each step. Returns the array at the end of the
+/// path (the whole `array` when `ancestor_ids` is empty).
+fn descend_by_ids(array: &ArrayRef, field: &Field, ancestor_ids: &[i32]) -> Result<ArrayRef> {
+    let mut arr = array.clone();
+    let mut fld = field;
+    for &id in ancestor_ids {
+        let child_pos = fld
+            .children
+            .iter()
+            .position(|c| c.id == id)
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "overlay descend: field id {id} not found under '{}'",
+                    fld.name
+                ))
+            })?;
+        let structs = arr.as_any().downcast_ref::<StructArray>().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay descend: expected a struct at '{}'",
+                fld.name
+            ))
+        })?;
+        arr = structs.column(child_pos).clone();
+        fld = &fld.children[child_pos];
+    }
+    Ok(arr)
+}
+
+/// Rebuild `array` with the array at `ancestor_ids` replaced by `new_atomic_field`, cloning
+/// the struct spine along the path and preserving each struct's null buffer and other
+/// children. With an empty path this is just `new_atomic_field` (whole-column replacement).
+fn splice_by_ids(
+    array: &ArrayRef,
+    field: &Field,
+    ancestor_ids: &[i32],
+    new_atomic_field: ArrayRef,
+) -> Result<ArrayRef> {
+    let Some((&id, rest)) = ancestor_ids.split_first() else {
+        return Ok(new_atomic_field);
+    };
+    let child_pos = field
+        .children
+        .iter()
+        .position(|c| c.id == id)
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay splice: field id {id} not found under '{}'",
+                field.name
+            ))
+        })?;
+    let structs = array
+        .as_any()
+        .downcast_ref::<StructArray>()
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay splice: expected a struct at '{}'",
+                field.name
+            ))
+        })?;
+    let len = structs.len();
+    let (fields, mut children, nulls) = structs.clone().into_parts();
+    children[child_pos] = splice_by_ids(
+        &children[child_pos],
+        &field.children[child_pos],
+        rest,
+        new_atomic_field,
+    )?;
+    Ok(Arc::new(StructArray::try_new_with_length(
+        fields, children, nulls, len,
+    )?))
+}
+
+/// Open the overlay readers a specific read needs and return the per-field plans to
+/// merge, pruned to that read.
+///
+/// `offsets_in_frag` are the rows the read will return. An overlay whose coverage is
+/// disjoint from those rows contributes nothing, so it is dropped and its file is
+/// never opened — a `take` that misses an overlay's cells pays no IO for it. Each
+/// surviving file is opened once, concurrently, projected to the covered fields; the
+/// value bytes are still not read here (the per-batch [`merge_overlay_batch`] fetches
+/// only the values it needs).
+pub async fn resolve_overlays(
+    planner: &OverlayReadPlanner,
+    offsets_in_frag: &[u32],
+    fragment: &FileFragment,
+    read_config: &FragReadConfig,
+) -> Result<Vec<LoadedAtomicField>> {
+    let read_offsets = read_offsets_bitmap(offsets_in_frag);
+
+    // A file is opened only if some atomic field it covers has cells among the requested rows.
+    // This is the row-selection pruning: overlays outside the read are skipped.
+    let mut file_needed = vec![false; planner.files.len()];
+    for atomic_field in &planner.atomic_fields {
+        for overlay in &atomic_field.overlays_newest_first {
+            if !overlay.coverage.is_disjoint(&read_offsets) {
+                file_needed[overlay.file] = true;
+            }
+        }
+    }
+
+    // Open each needed file once, concurrently. The reader is shared (via `Arc`) by
+    // every atomic field that file covers.
+    //
+    // These reads use priority 0 (highest): they are issued only when a ready
+    // consumer polls the batch task (see `merge_overlay_batch`), so we have already
+    // committed to reading this batch and the overlay reads cannot clog the
+    // backpressure queue ahead of work we are not ready for. (A future optimization
+    // could start the overlay fetches earlier to fill compute bubbles, which would
+    // want a priority tied to the base read.)
+    let opened: Vec<Option<Arc<dyn GenericFileReader>>> =
+        futures::future::try_join_all(planner.files.iter().enumerate().map(|(i, file)| {
+            let needed = file_needed[i];
+            async move {
+                if !needed {
+                    return Ok::<_, Error>(None);
+                }
+                Ok(fragment
+                    .open_reader(&file.data_file, Some(&file.open_projection), read_config)
+                    .await?
+                    .map(Arc::from))
+            }
+        }))
+        .await?;
+
+    let mut plans = Vec::new();
+    for atomic_field in &planner.atomic_fields {
+        let mut overlays_newest_first = Vec::new();
+        for overlay in &atomic_field.overlays_newest_first {
+            let Some(reader) = &opened[overlay.file] else {
+                continue; // pruned: coverage disjoint from the read
+            };
+            overlays_newest_first.push(LoadedAtomicFieldOverlay {
+                coverage: overlay.coverage.clone(),
+                reader: reader.clone(),
+            });
+        }
+        if !overlays_newest_first.is_empty() {
+            plans.push(LoadedAtomicField {
+                top_field: atomic_field.top_field.clone(),
+                ancestor_ids: atomic_field.ancestor_ids.clone(),
+                fetch_projection: atomic_field.fetch_projection.clone(),
+                overlays_newest_first,
+            });
+        }
+    }
+    Ok(plans)
+}
+
+/// The set of `offset_in_frag` a read will return, as a bitmap for cheap
+/// intersection against overlay coverages. Contiguous scans build a single range;
+/// arbitrary `take` offsets (small batches) are inserted individually.
+fn read_offsets_bitmap(offsets_in_frag: &[u32]) -> RoaringBitmap {
+    let mut bitmap = RoaringBitmap::new();
+    match contiguous_frag_start(offsets_in_frag) {
+        Some(start) => {
+            let end = (start as u64 + offsets_in_frag.len() as u64).min(u32::MAX as u64) as u32;
+            bitmap.insert_range(start..end);
+        }
+        None => bitmap.extend(offsets_in_frag.iter().copied()),
+    }
+    bitmap
+}
+
+/// Resolve overlays for one base batch: route each projected atomic field against the batch's
+/// `offsets_in_frag`, fetch only the overlay values the batch needs (concurrently with
+/// the base read), assemble the merged atomic field, and splice it into its output column.
+/// AtomicFields with no covered rows, and columns with no plan, pass through.
+pub async fn merge_overlay_batch(
+    base: ReadBatchFut,
+    offsets_in_frag: &[u32],
+    plans: &[LoadedAtomicField],
+) -> Result<RecordBatch> {
+    let atomic_field_work = futures::future::try_join_all(plans.iter().map(|plan| async move {
+        let coverages: Vec<&RoaringBitmap> = plan
+            .overlays_newest_first
+            .iter()
+            .map(|overlay| overlay.coverage.as_ref())
+            .collect();
+        let routing = route_overlays(offsets_in_frag, &coverages);
+        if !routing.any_overlay {
+            return Ok::<_, Error>((plan, None));
+        }
+        // Fetch each overlay's values and descend to the atomic field array. The fetch is
+        // projected to the atomic field's ancestor path, so the fetched column is the pruned
+        // top-level column; `descend_by_ids` walks it down to the atomic field.
+        let atomic_field = &plan.fetch_projection.fields[0];
+        let fetched = futures::future::try_join_all(
+            plan.overlays_newest_first
+                .iter()
+                .zip(&routing.offsets_in_overlay)
+                .map(|(overlay, offsets_in_overlay)| async move {
+                    let column = fetch_overlay_values(
+                        overlay.reader.as_ref(),
+                        plan.fetch_projection.clone(),
+                        offsets_in_overlay,
+                    )
+                    .await?;
+                    descend_by_ids(&column, atomic_field, &plan.ancestor_ids)
+                }),
+        )
+        .await?;
+        Ok((plan, Some((routing, fetched))))
+    }));
+
+    // The base read and every overlay value read proceed concurrently.
+    let (batch, resolved) = futures::future::try_join(base, atomic_field_work).await?;
+
+    let schema = batch.schema();
+    let mut columns = batch.columns().to_vec();
+    for (plan, work) in resolved {
+        let Some((routing, fetched)) = work else {
+            continue;
+        };
+        let Some(idx) = schema.index_of(&plan.top_field.name).ok() else {
+            // The plan's column is not in this batch's projection; skip it.
+            continue;
+        };
+        let base_atomic_field = descend_by_ids(&columns[idx], &plan.top_field, &plan.ancestor_ids)?;
+        let merged_atomic_field = assemble_overlay_column(&base_atomic_field, &routing, &fetched)?;
+        columns[idx] = splice_by_ids(
+            &columns[idx],
+            &plan.top_field,
+            &plan.ancestor_ids,
+            merged_atomic_field,
+        )?;
+    }
+    Ok(RecordBatch::try_new(schema, columns)?)
+}
+
+/// Fetch one overlay's values at the given `offsets_in_overlay` (sorted, unique):
+/// the corresponding entries of its value column, as the top-level column pruned to
+/// `projection`. Returns `offsets_in_overlay.len()` rows in the same order; empty
+/// input reads nothing and returns an empty column.
+async fn fetch_overlay_values(
+    reader: &dyn GenericFileReader,
+    projection: Arc<Schema>,
+    offsets_in_overlay: &[u32],
+) -> Result<ArrayRef> {
+    if offsets_in_overlay.is_empty() {
+        return Ok(arrow_array::new_empty_array(
+            &projection.fields[0].data_type(),
+        ));
+    }
+    let mut tasks = reader
+        .take_all_tasks(
+            offsets_in_overlay,
+            offsets_in_overlay.len() as u32,
+            projection,
+            None,
+        )
+        .await?;
+    let mut chunks: Vec<ArrayRef> = Vec::new();
+    while let Some(task) = tasks.next().await {
+        let batch = task.task.await?;
+        chunks.push(batch.column(0).clone());
+    }
+    let chunk_refs: Vec<&dyn arrow_array::Array> = chunks.iter().map(|a| a.as_ref()).collect();
+    Ok(arrow_select::concat::concat(&chunk_refs)?)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::{Int32Array, StringArray, UInt32Array};
+    use lance_table::format::overlay::OverlayCoverage;
+    use std::sync::Arc;
+
+    fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+        Arc::new(Int32Array::from_iter(values))
+    }
+
+    fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+        RoaringBitmap::from_iter(offsets)
+    }
+
+    fn flat_test_schema() -> Schema {
+        use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+
+        let mut schema = Schema::try_from(&ArrowSchema::new(
+            (0..5)
+                .map(|id| ArrowField::new(format!("field_{id}"), DataType::Int32, true))
+                .collect::<Vec<_>>(),
+        ))
+        .unwrap();
+        schema.set_field_id(None);
+        schema
+    }
+
+    /// Physical offsets for a contiguous range `[start, start + len)`.
+    fn offsets(start: u32, len: usize) -> Vec<u32> {
+        (start..start + len as u32).collect()
+    }
+
+    /// Drive the production flow purely in memory: route against the coverage
+    /// bitmaps, then fetch just the requested `offset_in_overlay` entries from each
+    /// overlay's *full* value column (exactly what the value-pushdown `take` does on
+    /// disk), then assemble. `overlays_newest_first` holds each overlay's
+    /// `(coverage, full value column indexed by offset_in_overlay)`.
+    fn resolve(
+        base: &ArrayRef,
+        offsets: &[u32],
+        overlays_newest_first: &[(RoaringBitmap, ArrayRef)],
+    ) -> ArrayRef {
+        let coverages: Vec<&RoaringBitmap> = overlays_newest_first.iter().map(|(c, _)| c).collect();
+        let routing = route_overlays(offsets, &coverages);
+        let fetched: Vec<ArrayRef> = overlays_newest_first
+            .iter()
+            .zip(&routing.offsets_in_overlay)
+            .map(|((_, full), offsets_in_overlay)| {
+                let indices = UInt32Array::from(offsets_in_overlay.clone());
+                arrow_select::take::take(full.as_ref(), &indices, None).unwrap()
+            })
+            .collect();
+        assemble_overlay_column(base, &routing, &fetched).unwrap()
+    }
+
+    fn assert_i32_eq(actual: &ArrayRef, expected: impl IntoIterator<Item = Option<i32>>) {
+        let actual = actual.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(actual, &Int32Array::from_iter(expected));
+    }
+
+    #[test]
+    fn test_no_overlays_returns_base() {
+        let base = i32_array([Some(1), Some(2), Some(3)]);
+        let resolved = resolve(&base, &offsets(0, 3), &[]);
+        assert_i32_eq(&resolved, [Some(1), Some(2), Some(3)]);
+    }
+
+    #[test]
+    fn test_single_overlay_value_offset() {
+        // Base ages [30, 25, 40, 22]; overlay sets offset_in_frag 1 -> 26, whose
+        // value sits at offset_in_overlay 0.
+        let base = i32_array([Some(30), Some(25), Some(40), Some(22)]);
+        let overlay = (bitmap([1]), i32_array([Some(26)]));
+        let resolved = resolve(&base, &offsets(0, 4), &[overlay]);
+        assert_i32_eq(&resolved, [Some(30), Some(26), Some(40), Some(22)]);
+    }
+
+    #[test]
+    fn test_value_offsets_multiple_cells() {
+        // Coverage {0, 2, 3} -> values at offset_in_overlay 0, 1, 2.
+        let base = i32_array([Some(10), Some(11), Some(12), Some(13)]);
+        let overlay = (
+            bitmap([0, 2, 3]),
+            i32_array([Some(100), Some(120), Some(130)]),
+        );
+        let resolved = resolve(&base, &offsets(0, 4), &[overlay]);
+        assert_i32_eq(&resolved, [Some(100), Some(11), Some(120), Some(130)]);
+    }
+
+    #[test]
+    fn test_newest_overlay_wins() {
+        // Two overlays both cover offset_in_frag 1; the newest (first in the slice)
+        // wins.
+        let base = i32_array([Some(0), Some(1), Some(2)]);
+        let newest = (bitmap([1]), i32_array([Some(999)]));
+        let older = (bitmap([1, 2]), i32_array([Some(111), Some(222)]));
+        let resolved = resolve(&base, &offsets(0, 3), &[newest, older]);
+        // offset 1 -> newest (999); offset 2 -> only older covers it (222).
+        assert_i32_eq(&resolved, [Some(0), Some(999), Some(222)]);
+    }
+
+    #[test]
+    fn test_null_override_vs_fall_through() {
+        // A covered offset with a NULL value overrides the cell to NULL; an
+        // absent offset falls through to the base.
+        let base = i32_array([Some(1), Some(2), Some(3)]);
+        let overlay = (bitmap([0]), i32_array([None]));
+        let resolved = resolve(&base, &offsets(0, 3), &[overlay]);
+        assert_i32_eq(&resolved, [None, Some(2), Some(3)]);
+    }
+
+    #[test]
+    fn test_physical_start_offset() {
+        // The batch covers physical rows [10, 13); the overlay covers offset 11.
+        let base = i32_array([Some(0), Some(0), Some(0)]);
+        let overlay = (bitmap([11]), i32_array([Some(7)]));
+        let resolved = resolve(&base, &offsets(10, 3), &[overlay]);
+        assert_i32_eq(&resolved, [Some(0), Some(7), Some(0)]);
+    }
+
+    #[test]
+    fn test_string_column_merge() {
+        let base: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"]));
+        let overlay = (
+            bitmap([0, 2]),
+            Arc::new(StringArray::from(vec!["A", "C"])) as ArrayRef,
+        );
+        let resolved = resolve(&base, &offsets(0, 3), &[overlay]);
+        let expected: ArrayRef = Arc::new(StringArray::from(vec!["A", "b", "C"]));
+        assert_eq!(&resolved, &expected);
+    }
+
+    #[test]
+    fn test_non_contiguous_offsets() {
+        // `take` supplies arbitrary, non-contiguous offsets_in_frag. The base rows
+        // correspond to offsets 5, 1, 8 (in that order); the overlay covers offsets
+        // {1, 8}, whose values sit at offset_in_overlay 0, 1.
+        let base = i32_array([Some(50), Some(10), Some(80)]);
+        let overlay = (bitmap([1, 8]), i32_array([Some(11), Some(88)]));
+        let resolved = resolve(&base, &[5, 1, 8], &[overlay]);
+        // offset 5 uncovered -> base 50; offset 1 -> offset_in_overlay 0 (11);
+        // offset 8 -> offset_in_overlay 1 (88).
+        assert_i32_eq(&resolved, [Some(50), Some(11), Some(88)]);
+    }
+
+    #[test]
+    fn test_routing_dedups_repeated_offsets() {
+        // A `take` may request the same offset twice; both rows must route to the
+        // same overlay value, and that value is fetched only once.
+        let coverage = bitmap([2, 5]);
+        let routing = route_overlays(&[5, 2, 5], &[&coverage]);
+        // offset_in_frag 5 is offset_in_overlay 1, offset_in_frag 2 is
+        // offset_in_overlay 0: distinct values {0, 1}, sorted.
+        assert_eq!(routing.offsets_in_overlay, vec![vec![0, 1]]);
+        let full = i32_array([Some(20), Some(50)]); // values at offset_in_overlay 0, 1
+        let fetched = vec![
+            arrow_select::take::take(
+                full.as_ref(),
+                &UInt32Array::from(routing.offsets_in_overlay[0].clone()),
+                None,
+            )
+            .unwrap(),
+        ];
+        let base = i32_array([Some(0), Some(0), Some(0)]);
+        let resolved = assemble_overlay_column(&base, &routing, &fetched).unwrap();
+        assert_i32_eq(&resolved, [Some(50), Some(20), Some(50)]);
+    }
+
+    #[test]
+    fn test_assemble_value_count_mismatch_errors() {
+        let coverage = bitmap([0, 1]);
+        let routing = route_overlays(&[0, 1], &[&coverage]);
+        let base = i32_array([Some(1), Some(2)]);
+        // One value supplied for two requested offsets is a caller bug.
+        let fetched = vec![i32_array([Some(9)])];
+        assert!(assemble_overlay_column(&base, &routing, &fetched).is_err());
+    }
+
+    #[test]
+    fn test_contiguous_fast_path_matches_general() {
+        // The contiguous fast path must produce byte-for-byte identical routing to
+        // the general offset-major path for any contiguous batch. Fuzz a range of
+        // fragment starts, lengths, overlay counts, and coverage densities —
+        // including bits outside the batch range — and compare both paths.
+        let mut state = 0x9e3779b97f4a7c15u64;
+        let mut next = || {
+            state = state
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407);
+            (state >> 33) as u32
+        };
+        for _ in 0..500 {
+            let frag_start = next() % 64;
+            let len = (next() % 48 + 1) as usize;
+            let num_overlays = (next() % 5) as usize;
+            let coverages: Vec<RoaringBitmap> = (0..num_overlays)
+                .map(|_| {
+                    let density = next() % 101;
+                    let mut b = RoaringBitmap::new();
+                    for off in frag_start.saturating_sub(3)..frag_start + len as u32 + 3 {
+                        if next() % 100 < density {
+                            b.insert(off);
+                        }
+                    }
+                    b
+                })
+                .collect();
+            let refs: Vec<&RoaringBitmap> = coverages.iter().collect();
+            let contiguous_offsets: Vec<u32> = (frag_start..frag_start + len as u32).collect();
+
+            let fast = route_contiguous(frag_start, len, &refs);
+            let general = route_arbitrary(&contiguous_offsets, &refs);
+            assert_eq!(fast.indices, general.indices, "indices differ");
+            assert_eq!(
+                fast.offsets_in_overlay, general.offsets_in_overlay,
+                "offsets_in_overlay differ"
+            );
+            assert_eq!(fast.any_overlay, general.any_overlay, "any_overlay differs");
+        }
+    }
+
+    /// `outer { middle { a, b } }` for exercising the descend/splice helpers.
+    fn nested_struct() -> (Schema, ArrayRef) {
+        use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
+        let mid = Fields::from(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+        ]);
+        let outer_fields =
+            Fields::from(vec![ArrowField::new("middle", DataType::Struct(mid), true)]);
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "outer",
+            DataType::Struct(outer_fields),
+            true,
+        )]);
+        let mut schema = Schema::try_from(&arrow_schema).unwrap();
+        schema.set_field_id(None);
+        let middle = StructArray::from(vec![
+            (
+                Arc::new(ArrowField::new("a", DataType::Int32, true)),
+                i32_array([Some(1), Some(2), Some(3)]),
+            ),
+            (
+                Arc::new(ArrowField::new("b", DataType::Int32, true)),
+                i32_array([Some(10), Some(20), Some(30)]),
+            ),
+        ]);
+        let outer: ArrayRef = Arc::new(StructArray::from(vec![(
+            Arc::new(ArrowField::new("middle", middle.data_type().clone(), true)),
+            Arc::new(middle) as ArrayRef,
+        )]));
+        (schema, outer)
+    }
+
+    #[test]
+    fn test_descend_and_splice_roundtrip() {
+        let (schema, outer_arr) = nested_struct();
+        let outer_field = &schema.fields[0];
+        let middle_id = outer_field.children[0].id;
+        let a_id = outer_field.children[0].children[0].id;
+        let path = [middle_id, a_id];
+
+        // Descend to the deep leaf `outer.middle.a`.
+        let a = descend_by_ids(&outer_arr, outer_field, &path).unwrap();
+        assert_i32_eq(&a, [Some(1), Some(2), Some(3)]);
+
+        // Splice a replacement in; only `a` changes, `b` is preserved.
+        let spliced = splice_by_ids(
+            &outer_arr,
+            outer_field,
+            &path,
+            i32_array([Some(7), Some(8), Some(9)]),
+        )
+        .unwrap();
+        let middle = spliced
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap()
+            .column(0)
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap()
+            .clone();
+        assert_i32_eq(&middle.column(0).clone(), [Some(7), Some(8), Some(9)]);
+        assert_i32_eq(&middle.column(1).clone(), [Some(10), Some(20), Some(30)]);
+    }
+
+    #[test]
+    fn test_splice_preserves_struct_nulls() {
+        use arrow_buffer::NullBuffer;
+        let (schema, base) = nested_struct();
+        let outer_field = &schema.fields[0];
+        // Rebuild `outer` with a null at row 1 (a null struct value).
+        let base = base.as_any().downcast_ref::<StructArray>().unwrap();
+        let (fields, children, _) = base.clone().into_parts();
+        let outer_arr: ArrayRef = Arc::new(
+            StructArray::try_new(
+                fields,
+                children,
+                Some(NullBuffer::from(vec![true, false, true])),
+            )
+            .unwrap(),
+        );
+        let path = [
+            outer_field.children[0].id,
+            outer_field.children[0].children[0].id,
+        ];
+        let spliced = splice_by_ids(
+            &outer_arr,
+            outer_field,
+            &path,
+            i32_array([Some(7), Some(8), Some(9)]),
+        )
+        .unwrap();
+        let spliced = spliced.as_any().downcast_ref::<StructArray>().unwrap();
+        // The outer struct's null buffer survives the splice.
+        assert!(!spliced.is_null(0));
+        assert!(spliced.is_null(1));
+        assert!(!spliced.is_null(2));
+    }
+
+    /// A dense overlay covering `offsets` for `field_ids`, committed at `version`.
+    fn dense_overlay(
+        field_ids: Vec<i32>,
+        offsets: impl IntoIterator<Item = u32>,
+        version: u64,
+    ) -> lance_table::format::overlay::DataOverlayFile {
+        DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", field_ids, None),
+            coverage: OverlayCoverage::dense(bitmap(offsets)),
+            committed_version: version,
+        }
+    }
+
+    #[test]
+    fn test_exclusion_offsets_version_gate() {
+        let schema = flat_test_schema();
+        // index built at version 5; only overlays committed > 5 are excluded.
+        let overlays = vec![
+            dense_overlay(vec![3], [0, 1], 4),
+            dense_overlay(vec![3], [2, 7], 6),
+        ];
+        let excluded = overlay_exclusion_offsets(&overlays, &[3], 5, &schema).unwrap();
+        assert_eq!(excluded, bitmap([2, 7]));
+        // An overlay exactly at the index version is already incorporated.
+        let overlays = vec![dense_overlay(vec![3], [9], 5)];
+        assert!(
+            overlay_exclusion_offsets(&overlays, &[3], 5, &schema)
+                .unwrap()
+                .is_empty()
+        );
+    }
+
+    #[test]
+    fn test_exclusion_offsets_is_field_aware() {
+        let schema = flat_test_schema();
+        // An overlay touching only an unrelated field excludes nothing.
+        let overlays = vec![dense_overlay(vec![2], [0, 1, 2], 9)];
+        assert!(
+            overlay_exclusion_offsets(&overlays, &[3], 1, &schema)
+                .unwrap()
+                .is_empty()
+        );
+        // The union spans only the indexed fields the overlay actually carries.
+        let overlays = vec![dense_overlay(vec![2, 3], [4], 9)];
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(),
+            bitmap([4])
+        );
+    }
+
+    #[test]
+    fn test_exclusion_offsets_matches_nested_fields() {
+        let (schema, _) = nested_struct();
+        let outer = &schema.fields[0];
+        let middle = &outer.children[0];
+        let a = &middle.children[0];
+        let b = &middle.children[1];
+
+        let overlays = vec![dense_overlay(vec![a.id], [1], 9)];
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[outer.id], 1, &schema).unwrap(),
+            bitmap([1])
+        );
+        assert!(
+            overlay_exclusion_offsets(&overlays, &[b.id], 1, &schema)
+                .unwrap()
+                .is_empty()
+        );
+
+        let overlays = vec![dense_overlay(vec![middle.id], [2], 9)];
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[a.id], 1, &schema).unwrap(),
+            bitmap([2])
+        );
+    }
+
+    #[test]
+    fn test_exclusion_offsets_sparse_per_field() {
+        let schema = flat_test_schema();
+        // Sparse overlay: field 2 covers {2,3}, field 4 covers {1}.
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None),
+            coverage: OverlayCoverage::sparse(vec![bitmap([2, 3]), bitmap([1])]),
+            committed_version: 9,
+        };
+        let overlays = vec![overlay];
+        // Only the bitmap for the indexed field (4) contributes.
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[4], 1, &schema).unwrap(),
+            bitmap([1])
+        );
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[2], 1, &schema).unwrap(),
+            bitmap([2, 3])
+        );
+    }
+
+    #[test]
+    fn test_exclusion_offsets_unions_multiple_overlays() {
+        let schema = flat_test_schema();
+        let overlays = vec![
+            dense_overlay(vec![3], [1], 6),
+            dense_overlay(vec![3], [4, 5], 7),
+        ];
+        assert_eq!(
+            overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(),
+            bitmap([1, 4, 5])
+        );
+    }
+
+    /// An index segment covering `fields`, built at `dataset_version`, with the given
+    /// fragment coverage (`None` = legacy index predating fragment-bitmap tracking).
+    fn segment(
+        fields: Vec<i32>,
+        dataset_version: u64,
+        fragment_bitmap: Option<RoaringBitmap>,
+    ) -> IndexMetadata {
+        IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            name: "idx".into(),
+            fields,
+            dataset_version,
+            fragment_bitmap,
+            index_details: None,
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        }
+    }
+
+    fn fragment_with_overlay(id: u64, overlay: DataOverlayFile) -> Fragment {
+        let mut fragment = Fragment::new(id);
+        fragment.overlays.push(overlay);
+        fragment
+    }
+
+    #[test]
+    fn test_collect_frags_missing_bitmap_covers_all() {
+        let schema = flat_test_schema();
+        // A segment with no fragment_bitmap (legacy index predating bitmap tracking) must treat
+        // every overlaid fragment as covered so stale rows can't leak past the index unmasked.
+        let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9));
+        let overlaid: HashMap<u32, &Fragment> = HashMap::from([(3u32, &fragment)]);
+
+        let mut stale = RoaringBitmap::new();
+        collect_overlay_stale_frags(&segment(vec![3], 1, None), &overlaid, &mut stale, &schema)
+            .unwrap();
+        assert_eq!(stale, bitmap([3]), "missing bitmap must cover fragment 3");
+
+        // A present bitmap that excludes fragment 3 leaves it untouched.
+        let mut stale = RoaringBitmap::new();
+        collect_overlay_stale_frags(
+            &segment(vec![3], 1, Some(bitmap([0]))),
+            &overlaid,
+            &mut stale,
+            &schema,
+        )
+        .unwrap();
+        assert!(
+            stale.is_empty(),
+            "fragment absent from bitmap is not covered"
+        );
+
+        // A present bitmap that includes fragment 3 marks it stale.
+        let mut stale = RoaringBitmap::new();
+        collect_overlay_stale_frags(
+            &segment(vec![3], 1, Some(bitmap([3]))),
+            &overlaid,
+            &mut stale,
+            &schema,
+        )
+        .unwrap();
+        assert_eq!(stale, bitmap([3]));
+    }
+
+    #[test]
+    fn test_collect_rows_missing_bitmap_covers_all() {
+        let schema = flat_test_schema();
+        // Same covers-all guarantee at row-level granularity.
+        let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9));
+        let overlaid: HashMap<u32, &Fragment> = HashMap::from([(3u32, &fragment)]);
+
+        let mut stale = HashMap::new();
+        collect_overlay_stale_rows_for_segment(
+            &segment(vec![3], 1, None),
+            &overlaid,
+            &mut stale,
+            &schema,
+        )
+        .unwrap();
+        assert_eq!(
+            stale.get(&3),
+            Some(&bitmap([1, 2])),
+            "missing bitmap must cover fragment 3"
+        );
+
+        // A present bitmap that excludes fragment 3 yields no stale rows.
+        let mut stale = HashMap::new();
+        collect_overlay_stale_rows_for_segment(
+            &segment(vec![3], 1, Some(bitmap([0]))),
+            &overlaid,
+            &mut stale,
+            &schema,
+        )
+        .unwrap();
+        assert!(
+            stale.is_empty(),
+            "fragment absent from bitmap contributes no rows"
+        );
+    }
+}
diff --git a/rust/lance/src/dataset/refs.rs b/rust/lance/src/dataset/refs.rs
index 98b4f0cbc0a..0d3f65f7959 100644
--- a/rust/lance/src/dataset/refs.rs
+++ b/rust/lance/src/dataset/refs.rs
@@ -21,7 +21,6 @@ use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::fmt;
 use std::fmt::Formatter;
-use std::io::ErrorKind;
 use uuid::Uuid;
 
 pub const MAIN_BRANCH: &str = "main";
@@ -476,7 +475,7 @@ impl Branches<'_> {
 
         if !self.object_store().exists(&manifest_file.path).await? {
             return Err(Error::VersionNotFound {
-                message: format!("Manifest file {} does not exist", &manifest_file.path),
+                message: format!("Manifest file {} does not exist", manifest_file.path),
             });
         };
 
@@ -611,16 +610,8 @@ impl Branches<'_> {
             && let Err(e) = self.refs.object_store.remove_dir_all(delete_path).await
         {
             match &e {
-                Error::IO { source, .. } => {
-                    if let Some(io_err) = source.downcast_ref::<std::io::Error>() {
-                        if io_err.kind() == ErrorKind::NotFound {
-                            log::debug!("Branch directory already deleted: {}", io_err);
-                        } else {
-                            return Err(e);
-                        }
-                    } else {
-                        return Err(e);
-                    }
+                Error::NotFound { .. } => {
+                    log::debug!("Branch directory already deleted");
                 }
                 _ => return Err(e),
             }
diff --git a/rust/lance/src/dataset/rowids.rs b/rust/lance/src/dataset/rowids.rs
index c57d75386db..082b041d40a 100644
--- a/rust/lance/src/dataset/rowids.rs
+++ b/rust/lance/src/dataset/rowids.rs
@@ -6,6 +6,7 @@ use crate::session::caches::{RowIdIndexKey, RowIdSequenceKey};
 use crate::{Error, Result};
 use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt};
 use lance_core::utils::deletion::DeletionVector;
+use lance_select::{RowAddrSelection, RowAddrTreeMap};
 use lance_table::{
     format::{Fragment, RowIdMeta},
     rowids::{FragmentRowIdIndex, RowIdIndex, RowIdSequence, read_row_ids},
@@ -87,6 +88,132 @@ pub async fn get_row_id_index(
     }
 }
 
+/// Map a set of physical row addresses to their stable row ids
+///
+/// For each fragment present in `addrs`, the live rows in physical order carry
+/// the stable ids yielded by the fragment's [`RowIdSequence`] in the same
+/// order. Zipping the two (skipping deleted physical offsets) gives the
+/// `physical offset -> stable id` mapping. Addresses that point at deleted rows
+/// have no live counterpart and are dropped, which is correct: those rows are
+/// not part of the answer.
+pub(crate) async fn translate_addr_treemap_to_row_ids(
+    dataset: &Dataset,
+    addrs: &RowAddrTreeMap,
+) -> Result<RowAddrTreeMap> {
+    let mut row_ids = RowAddrTreeMap::new();
+    for (fragment_id, selection) in addrs.iter() {
+        let file_fragment = dataset.get_fragment(*fragment_id as usize).ok_or_else(|| {
+            Error::internal(format!(
+                "fragment {fragment_id} referenced by an address-domain index result \
+                 was not found in the dataset"
+            ))
+        })?;
+        let sequence = load_row_id_sequence(dataset, file_fragment.metadata()).await?;
+
+        match selection {
+            RowAddrSelection::Full => {
+                // The whole fragment is selected: every live row's id qualifies.
+                row_ids |= RowAddrTreeMap::from(sequence.as_ref());
+            }
+            RowAddrSelection::Partial(offsets) => {
+                let Some(max_offset) = offsets.max() else {
+                    continue;
+                };
+                let (deletion_vector, num_physical_rows) = futures::try_join!(
+                    file_fragment.get_deletion_vector(),
+                    file_fragment.physical_rows()
+                )?;
+                let num_physical_rows = u32::try_from(num_physical_rows).map_err(|_| {
+                    Error::internal(format!(
+                        "fragment_id={fragment_id} has num_physical_rows={num_physical_rows}, \
+                         which exceeds the maximum representable physical offset"
+                    ))
+                })?;
+                if max_offset >= num_physical_rows {
+                    return Err(Error::internal(format!(
+                        "fragment_id={fragment_id} selection has max_offset={max_offset}, \
+                         but num_physical_rows={num_physical_rows}"
+                    )));
+                }
+                let mut ids = sequence.iter();
+                for physical_offset in 0..num_physical_rows {
+                    if physical_offset > max_offset {
+                        break;
+                    }
+                    let deleted = deletion_vector
+                        .as_ref()
+                        .is_some_and(|dv| dv.contains(physical_offset));
+                    if deleted {
+                        continue;
+                    }
+                    match ids.next() {
+                        Some(id) => {
+                            if offsets.contains(physical_offset) {
+                                row_ids.insert(id);
+                            }
+                        }
+                        // The sequence yields one id per live row, so it can only
+                        // run dry before `max_offset` if it holds fewer ids than the
+                        // fragment has live rows. Breaking would silently drop the
+                        // remaining selected offsets and let stale index results
+                        // escape masking, so treat the mismatch as corruption.
+                        None => {
+                            return Err(Error::internal(format!(
+                                "fragment_id={fragment_id} row-id sequence exhausted at \
+                                 physical_offset={physical_offset} before reaching \
+                                 max_offset={max_offset} (num_physical_rows={num_physical_rows})"
+                            )));
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(row_ids)
+}
+
+/// Resolve row addresses to row ids, positionally: `addr = (fragment << 32) | offset`
+/// looks up `offset` in the fragment's [`RowIdSequence`]. The inverse companion of
+/// [`get_row_id_index`].
+///
+/// Returns one entry per input address, in order. Addresses that no longer
+/// resolve — a missing fragment, an out-of-range offset, or a `None` input —
+/// yield `None`. On datasets without stable row ids, addresses are the row
+/// ids, so the input is returned unchanged.
+pub async fn row_addrs_to_row_ids(
+    dataset: &Dataset,
+    addrs: impl IntoIterator<Item = Option<u64>>,
+) -> Result<Vec<Option<u64>>> {
+    let addrs: Vec<Option<u64>> = addrs.into_iter().collect();
+    if !dataset.manifest.uses_stable_row_ids() {
+        return Ok(addrs);
+    }
+
+    let mut positions_by_fragment: std::collections::HashMap<u32, Vec<usize>> =
+        std::collections::HashMap::new();
+    for (position, addr) in addrs.iter().enumerate() {
+        if let Some(addr) = addr {
+            positions_by_fragment
+                .entry((addr >> 32) as u32)
+                .or_default()
+                .push(position);
+        }
+    }
+
+    let mut ids: Vec<Option<u64>> = vec![None; addrs.len()];
+    for (fragment_id, positions) in positions_by_fragment {
+        let Some(fragment) = dataset.get_fragment(fragment_id as usize) else {
+            continue;
+        };
+        let sequence = load_row_id_sequence(dataset, fragment.metadata()).await?;
+        for position in positions {
+            let offset = addrs[position].expect("grouped from Some") as u32;
+            ids[position] = sequence.get(offset as usize);
+        }
+    }
+    Ok(ids)
+}
+
 async fn load_row_id_index(dataset: &Dataset) -> Result<lance_table::rowids::RowIdIndex> {
     let sequences = load_row_id_sequences(dataset, &dataset.manifest.fragments)
         .try_collect::<Vec<_>>()
@@ -243,6 +370,51 @@ mod test {
         assert_eq!(dataset.manifest().next_row_id, num_rows);
     }
 
+    #[tokio::test]
+    async fn test_row_addrs_to_row_ids() {
+        let num_rows = 25u64;
+        let batch = sequence_batch(0..num_rows as i32);
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let write_params = WriteParams {
+            enable_stable_row_ids: true,
+            max_rows_per_file: 10,
+            ..Default::default()
+        };
+        let dataset = Dataset::write(reader, "memory://", Some(write_params))
+            .await
+            .unwrap();
+
+        // Sequential assignment: row n lives at (n / 10, n % 10) with id n
+        let addr = |frag: u64, offset: u64| Some((frag << 32) | offset);
+        let addrs = vec![
+            addr(2, 1),   // id 21
+            addr(0, 3),   // id 3
+            None,         // null stays null
+            addr(0, 3),   // duplicates allowed
+            addr(9, 0),   // missing fragment
+            addr(1, 100), // out-of-range offset
+        ];
+        let ids = row_addrs_to_row_ids(&dataset, addrs).await.unwrap();
+        assert_eq!(ids, vec![Some(21), Some(3), None, Some(3), None, None]);
+
+        // Without stable row ids, addresses are the ids: input passes through
+        let batch = sequence_batch(0..num_rows as i32);
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let plain = Dataset::write(
+            reader,
+            "memory://",
+            Some(WriteParams {
+                max_rows_per_file: 10,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let addrs = vec![addr(1, 2), None];
+        let ids = row_addrs_to_row_ids(&plain, addrs.clone()).await.unwrap();
+        assert_eq!(ids, addrs);
+    }
+
     #[tokio::test]
     async fn test_row_ids_overwrite() {
         // Validate we don't re-use after overwriting
@@ -467,6 +639,53 @@ mod test {
         assert_eq!(index.get(5), None);
     }
 
+    /// 100 sequential rows across 4 fragments with every third row deleted.
+    async fn deleted_thirds_dataset(enable_stable_row_ids: bool) -> Dataset {
+        let batch = sequence_batch(0..100);
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let write_params = WriteParams {
+            enable_stable_row_ids,
+            max_rows_per_file: 25,
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write(reader, "memory://", Some(write_params))
+            .await
+            .unwrap();
+        dataset.delete("id % 3 = 0").await.unwrap();
+        dataset
+    }
+
+    #[rstest::rstest]
+    #[case::interleaved((0..100).collect(), (0..100).filter(|id| id % 3 != 0).collect())]
+    #[case::all_deleted(vec![0, 3, 9], vec![])]
+    #[case::all_live(vec![1, 2, 50, 98], vec![1, 2, 50, 98])]
+    #[tokio::test]
+    async fn test_filter_deleted_ids_with_stable_row_ids(
+        #[case] ids: Vec<u64>,
+        #[case] expected: Vec<u64>,
+    ) {
+        // Regression test for https://github.com/lance-format/lance/issues/7701:
+        // filter_deleted_ids must return exactly the live ids, in order.
+        // Sequential inserts, so stable row id == id column value.
+        let dataset = deleted_thirds_dataset(true).await;
+        assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected);
+    }
+
+    #[tokio::test]
+    async fn test_filter_deleted_ids_without_stable_row_ids() {
+        // Non-stable: ids are row addresses; only deleted rows drop out.
+        let dataset = deleted_thirds_dataset(false).await;
+
+        // Row addresses: fragment (id / 25) << 32 | offset (id % 25).
+        let addr = |id: u64| (id / 25) << 32 | (id % 25);
+        let ids = (0..100).map(addr).collect::<Vec<u64>>();
+        let expected = (0..100)
+            .filter(|id| id % 3 != 0)
+            .map(addr)
+            .collect::<Vec<u64>>();
+        assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected);
+    }
+
     fn build_rowid_to_i_map(row_ids: &UInt64Array, i_array: &Int32Array) -> HashMap<u64, i32> {
         row_ids
             .values()
diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs
index 671a2c24333..89987a17f9f 100644
--- a/rust/lance/src/dataset/scanner.rs
+++ b/rust/lance/src/dataset/scanner.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 
 use datafusion::config::ConfigOptions;
 use lance_select::result::IndexExprResultWireFormat;
@@ -17,7 +17,7 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaR
 use arrow_select::concat::concat_batches;
 use async_recursion::async_recursion;
 use chrono::Utc;
-use datafusion::common::{DFSchema, JoinType, NullEquality, SchemaExt, exec_datafusion_err};
+use datafusion::common::{DFSchema, JoinType, NullEquality, exec_datafusion_err};
 use datafusion::functions_aggregate;
 use datafusion::logical_expr::{Expr, ScalarUDF, col, lit};
 use datafusion::physical_expr::PhysicalSortExpr;
@@ -83,15 +83,21 @@ use tracing::{Span, info_span, instrument};
 use uuid::Uuid;
 
 use super::Dataset;
+use crate::dataset::overlay::{
+    collect_overlay_stale_frags, collect_overlay_stale_rows_for_segment, overlaid_fragments,
+};
 use crate::dataset::row_offsets_to_row_addresses;
+use crate::dataset::rowids::translate_addr_treemap_to_row_ids;
 use crate::dataset::utils::SchemaAdapter;
 use crate::index::DatasetIndexInternalExt;
 use crate::index::scalar::inverted::{load_segment_details, load_segments};
-use crate::index::scalar_logical::scalar_index_fragment_bitmap;
+use crate::index::scalar_logical::{load_named_scalar_segments, scalar_index_fragment_bitmap};
 use crate::index::vector::utils::{
     default_distance_type_for, get_vector_dim, get_vector_type, validate_distance_type_for,
 };
-use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions};
+use crate::io::exec::filtered_read::{
+    FilteredReadExec, FilteredReadOptions, FilteredReadThreadingMode,
+};
 use crate::io::exec::fts::{
     BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec,
 };
@@ -101,7 +107,8 @@ use crate::io::exec::{
     AddRowAddrExec, FilterPlan as ExprFilterPlan, KNNVectorDistanceExec, LancePushdownScanExec,
     LanceScanExec, Planner, PreFilterSource, ScanConfig, TakeExec,
     knn::{
-        KnnBatchParams, QUERY_INDEX_COL, knn_empty_result_schema, new_knn_exec, query_index_field,
+        KnnBatchParams, QUERY_INDEX_COL, knn_empty_result_schema, new_knn_batch_exec, new_knn_exec,
+        query_index_field,
     },
     project,
 };
@@ -119,7 +126,9 @@ pub use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts}
 #[cfg(feature = "substrait")]
 use lance_datafusion::substrait::parse_substrait;
 
-pub(crate) const BATCH_SIZE_FALLBACK: usize = 8192;
+/// Rows per output batch when neither the scan options nor
+/// `LANCE_DEFAULT_BATCH_SIZE` specify one.
+pub const BATCH_SIZE_FALLBACK: usize = 8192;
 
 /// Parse an environment variable as a specific type, logging a warning on parse failure.
 fn parse_env_var<T: std::str::FromStr>(env_var_name: &str, default_val: &str) -> Option<T>
@@ -1371,8 +1380,12 @@ impl Scanner {
         self
     }
 
-    /// Set the prefetch size.
-    /// Ignored in v2 and newer format
+    /// Set the number of batches to decode concurrently.
+    ///
+    /// This bounds the decode fan-out of the scan: at most this many batch-decode
+    /// tasks run in flight at once. Defaults to `get_num_compute_intensive_cpus()`.
+    ///
+    /// `nbatches` must be greater than zero.
     pub fn batch_readahead(&mut self, nbatches: usize) -> &mut Self {
         self.batch_readahead = nbatches;
         self
@@ -1742,7 +1755,7 @@ impl Scanner {
                     .field(&column.column_name)
                     .ok_or(Error::invalid_input(format!(
                         "Column {} not found",
-                        &column.column_name
+                        column.column_name
                     )))?;
             }
         }
@@ -1760,8 +1773,9 @@ impl Scanner {
 
     /// Configure the speed / accuracy tradeoff for approximate vector search.
     ///
-    /// This setting is currently only used by RQ-quantized indexes, such as
-    /// IVF_RQ. Other index types ignore this setting.
+    /// This setting is currently used by RQ-quantized indexes (such as
+    /// IVF_RQ) and by prefiltered search on HNSW indexes, where `Fast`
+    /// enables the ACORN traversal. Other index types ignore this setting.
     pub fn approx_mode(&mut self, approx_mode: ApproxMode) -> &mut Self {
         if let Some(q) = self.nearest.as_mut() {
             q.approx_mode = approx_mode;
@@ -1899,6 +1913,38 @@ impl Scanner {
         }
     }
 
+    /// Ensure `input` exposes `column_name` as a top-level column.
+    ///
+    /// Nested FTS flat-search paths read the projected struct column from storage
+    /// but the FTS executor consumes a single document column by name.
+    fn ensure_column_alias(
+        &self,
+        input: Arc<dyn ExecutionPlan>,
+        column_name: &str,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let input_schema = input.schema();
+        if input_schema.column_with_name(column_name).is_some() {
+            return Ok(input);
+        }
+
+        let mut projection_exprs = Vec::with_capacity(input_schema.fields().len() + 1);
+        for field in input_schema.fields() {
+            projection_exprs.push((
+                Arc::new(Column::new_with_schema(
+                    field.name(),
+                    input_schema.as_ref(),
+                )?) as Arc<dyn PhysicalExpr>,
+                field.name().clone(),
+            ));
+        }
+        projection_exprs.push((
+            Self::create_column_expr(column_name, self.dataset.as_ref(), input_schema.as_ref())?,
+            column_name.to_string(),
+        ));
+
+        Ok(Arc::new(ProjectionExec::try_new(projection_exprs, input)?))
+    }
+
     /// Set whether to use statistics to optimize the scan (default: true)
     ///
     /// This is used for debugging or benchmarking purposes.
@@ -2186,6 +2232,9 @@ impl Scanner {
     }
 
     #[allow(clippy::type_complexity)]
+    // TODO(datafusion-54): migrate off the deprecated
+    // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder.
+    #[allow(deprecated)]
     fn build_physical_aggregate_expr(
         &self,
         expr: &Expr,
@@ -2310,11 +2359,12 @@ impl Scanner {
             MaterializationStyle::AllLate => false,
             MaterializationStyle::AllEarlyExcept(ref cols) => !cols.contains(&(field.id as u32)),
             MaterializationStyle::Heuristic => {
-                if field.is_blob() {
-                    // By default, blobs are loaded as descriptions, and so should be early
-                    //
-                    // TODO: Once we make blob handling configurable, we should use the blob
-                    // handling setting here.
+                if field.is_blob() && self.blob_handling.returns_description(field) {
+                    // A blob returned as a description (offset + size) is tiny, so it is
+                    // cheaper to read eagerly. When blob_handling materializes the full
+                    // binary value instead (e.g. `all_binary`), fall through to the
+                    // width-based heuristic so a selective filter can late-materialize it
+                    // rather than reading the whole column.
                     return true;
                 }
 
@@ -2362,6 +2412,12 @@ impl Scanner {
     }
 
     fn validate_options(&self) -> Result<()> {
+        if self.batch_readahead == 0 {
+            return Err(Error::invalid_input_source(
+                "batch_readahead must be greater than 0, got 0".into(),
+            ));
+        }
+
         if self.include_deleted_rows && !self.projection_plan.physical_projection.with_row_id {
             return Err(Error::invalid_input_source(
                 "include_deleted_rows is set but with_row_id is false".into(),
@@ -2858,6 +2914,8 @@ impl Scanner {
         fragments: Option<Arc<Vec<Fragment>>>,
         scan_range: Option<Range<u64>>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        // Kept for the overlay stale-Take path below, which re-evaluates blocked stale rows.
+        let user_projection = projection.clone();
         let mut read_options = FilteredReadOptions::basic_full_read(&self.dataset)
             .with_filter_plan(filter_plan.clone())
             .with_projection(projection);
@@ -2874,6 +2932,11 @@ impl Scanner {
             read_options = read_options.with_batch_size(batch_size as u32);
         }
 
+        // Bound the decode fan-out by `batch_readahead`.
+        read_options = read_options.with_threading_mode(
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(self.batch_readahead),
+        );
+
         if let Some(file_reader_options) = self.resolved_file_reader_options() {
             read_options = read_options.with_file_reader_options(file_reader_options);
         }
@@ -2894,6 +2957,24 @@ impl Scanner {
             read_options = read_options.with_only_indexed_fragments();
         }
 
+        // Mask data overlay files: a row with an overlay committed after an index it relies on
+        // touched an indexed field can no longer be trusted to that index. Block just those rows
+        // from the index result (their fragments stay indexed, so non-stale rows keep the index)
+        // and re-evaluate them on a targeted take path below — O(stale_rows), not O(fragment).
+        let mut overlay_stale_rows: HashMap<u32, RoaringBitmap> = HashMap::new();
+        if let Some(index_query) = filter_plan.index_query.as_ref() {
+            let candidate_frags = read_options
+                .fragments
+                .clone()
+                .unwrap_or_else(|| self.dataset.fragments().clone());
+            overlay_stale_rows = self
+                .overlay_stale_index_rows(index_query, &candidate_frags)
+                .await?;
+            if let Some(block) = self.stale_rows_block_mask(&overlay_stale_rows).await? {
+                read_options = read_options.with_overlay_block(block);
+            }
+        }
+
         let result_format = self.index_expr_result_format();
         let index_input = filter_plan.index_query.clone().map(|index_query| {
             Arc::new(ScalarIndexExec::new(
@@ -2903,10 +2984,36 @@ impl Scanner {
             )) as Arc<dyn ExecutionPlan>
         });
 
-        Ok(Arc::new(FilteredReadExec::try_new(
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(FilteredReadExec::try_new(
             self.dataset.clone(),
             read_options,
             index_input,
+        )?);
+
+        if overlay_stale_rows.is_empty() {
+            return Ok(plan);
+        }
+
+        // Stale-Take path: take the stale rows' current (overlay-merged) values and re-apply the
+        // full filter, then union with the indexed read. These rows were blocked from the index
+        // result above, so this is the only path that can surface them.
+        let filter = filter_plan.full_expr.as_ref().expect_ok()?;
+        let filter_cols = Planner::column_names_in_expr(filter);
+        let take_projection = user_projection.union_columns(filter_cols, OnMissing::Error)?;
+
+        let stale_node = self
+            .stale_rows_take(&overlay_stale_rows, take_projection)
+            .await?;
+        let planner = Planner::new(stale_node.schema());
+        let optimized_filter = planner.optimize_expr(filter.clone())?;
+        let filtered = Arc::new(LanceFilterExec::try_new(optimized_filter, stale_node)?);
+        let stale_path: Arc<dyn ExecutionPlan> =
+            Arc::new(project(filtered, plan.schema().as_ref())?);
+
+        let unioned = UnionExec::try_new(vec![plan, stale_path])?;
+        Ok(Arc::new(RepartitionExec::try_new(
+            unioned,
+            datafusion::physical_plan::Partitioning::RoundRobinBatch(1),
         )?))
     }
 
@@ -3274,6 +3381,10 @@ impl Scanner {
                 self.fragments_covered_by_fts_query(&query).await?,
             )
             .await?;
+        // Data overlay masking: match queries drop stale segments and re-evaluate the affected
+        // fragments on the flat-text path (`plan_match_query` / `fts_stale_frags_and_fresh_segments`),
+        // and phrase queries exclude stale segments (`plan_phrase_query`). Both keep stale index
+        // hits out of the result.
         let fts_exec = self
             .plan_fts(&query, &params, filter_plan, &prefilter_source)
             .await?;
@@ -3466,12 +3577,40 @@ impl Scanner {
                 .to_string()));
         }
 
-        Ok(Arc::new(PhraseQueryExec::new(
-            self.dataset.clone(),
-            query.clone(),
-            params.clone(),
-            prefilter_source.clone(),
-        )))
+        // Mask data overlay files: a fragment with an overlay committed after this FTS index can
+        // no longer be trusted to its inverted-index positions, so a stale phrase could still
+        // match. Exclude any segment covering such a fragment. Unlike match queries, phrase
+        // queries have no flat re-evaluation path, so — exactly as for unindexed fragments, which
+        // phrase queries already do not search — overlaid fragments are simply dropped from the
+        // phrase result; new phrase matches there surface once compaction folds the overlay into
+        // the base. This removes stale hits without introducing wrong ones.
+        let target_fragments = self
+            .fragments
+            .clone()
+            .unwrap_or_else(|| self.dataset.fragments().to_vec());
+        let (_flat_frag_ids, fresh_segments) = self
+            .fts_stale_frags_and_fresh_segments(&column, &target_fragments)
+            .await?;
+
+        let exec: Arc<dyn ExecutionPlan> = match fresh_segments {
+            // Every segment covers a stale fragment: with no flat phrase path there is nothing
+            // trustworthy left to search, so the phrase query returns no rows.
+            Some(segs) if segs.is_empty() => Arc::new(EmptyExec::new(FTS_SCHEMA.clone())),
+            Some(segs) => Arc::new(PhraseQueryExec::new_with_segments(
+                self.dataset.clone(),
+                query.clone(),
+                params.clone(),
+                prefilter_source.clone(),
+                segs,
+            )),
+            None => Arc::new(PhraseQueryExec::new(
+                self.dataset.clone(),
+                query.clone(),
+                params.clone(),
+                prefilter_source.clone(),
+            )),
+        };
+        Ok(exec)
     }
 
     async fn plan_match_query(
@@ -3506,30 +3645,62 @@ impl Scanner {
                 let unindexed_fragments = self
                     .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?);
 
-                // If all target fragments are unindexed, skip index entirely
-                if unindexed_fragments.len() == target_fragments.len() {
+                // Fragments whose FTS index entries may be stale due to a newer data overlay.
+                // These are excluded from the indexed path and re-evaluated on the flat path.
+                let (stale_flat_frag_ids, fresh_segments) = self
+                    .fts_stale_frags_and_fresh_segments(&column, &target_fragments)
+                    .await?;
+
+                // Fragments that need flat evaluation: unindexed + stale (deduplicated).
+                let flat_fragments: Vec<Fragment> = {
+                    let mut seen = RoaringBitmap::new();
+                    let mut frags = Vec::new();
+                    for f in unindexed_fragments.iter().chain(
+                        target_fragments
+                            .iter()
+                            .filter(|f| stale_flat_frag_ids.contains(f.id as u32)),
+                    ) {
+                        if seen.insert(f.id as u32) {
+                            frags.push(f.clone());
+                        }
+                    }
+                    frags
+                };
+
+                // If all target fragments need flat evaluation, skip the indexed path.
+                if flat_fragments.len() == target_fragments.len() {
                     if self.fast_search {
                         return Ok(Arc::new(EmptyExec::new(FTS_SCHEMA.clone())));
                     }
                     let flat_match_plan = self
-                        .plan_flat_match_query(unindexed_fragments, query, params, filter_plan)
+                        .plan_flat_match_query(flat_fragments, query, params, filter_plan)
                         .await?;
                     return Ok(flat_match_plan);
                 }
 
-                // Mixed case: use index + flat search for unindexed
-                let match_plan: Arc<dyn ExecutionPlan> = Arc::new(MatchQueryExec::new(
-                    self.dataset.clone(),
-                    query.clone(),
-                    params.clone(),
-                    prefilter_source.clone(),
-                ));
+                // Build the indexed path. When overlays made some segments stale we use
+                // `new_with_segments` to restrict the search to fresh segments only.
+                let match_plan: Arc<dyn ExecutionPlan> = match fresh_segments {
+                    Some(segs) => Arc::new(MatchQueryExec::new_with_segments(
+                        self.dataset.clone(),
+                        query.clone(),
+                        params.clone(),
+                        prefilter_source.clone(),
+                        segs,
+                    )),
+                    None => Arc::new(MatchQueryExec::new(
+                        self.dataset.clone(),
+                        query.clone(),
+                        params.clone(),
+                        prefilter_source.clone(),
+                    )),
+                };
 
-                if self.fast_search || unindexed_fragments.is_empty() {
+                if self.fast_search || flat_fragments.is_empty() {
                     (Some(match_plan), None)
                 } else {
                     let flat_match_plan = self
-                        .plan_flat_match_query(unindexed_fragments, query, params, filter_plan)
+                        .plan_flat_match_query(flat_fragments, query, params, filter_plan)
                         .await?;
                     (Some(match_plan), Some(flat_match_plan))
                 }
@@ -3587,7 +3758,7 @@ impl Scanner {
             ))?
             .clone();
 
-        let mut columns = vec![column];
+        let mut columns = vec![column.clone()];
         if let Some(refine_expr) = filter_plan.refine_expr.as_ref() {
             columns.extend(Planner::column_names_in_expr(refine_expr));
         }
@@ -3611,6 +3782,7 @@ impl Scanner {
         if let Some(refine_expr) = filter_plan.refine_expr.as_ref() {
             plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?);
         }
+        plan = self.ensure_column_alias(plan, &column)?;
 
         let flat_match_plan = Arc::new(FlatMatchQueryExec::new(
             self.dataset.clone(),
@@ -3771,7 +3943,16 @@ impl Scanner {
 
         if let Some((index_name, index_segments, index_metric)) = index_and_segments {
             if self.is_batch_nearest {
-                return self.batch_indexed_vector_search(filter_plan, &q).await;
+                validate_distance_type_for(index_metric, &element_type)?;
+                return self
+                    .batch_indexed_vector_search(
+                        filter_plan,
+                        &q,
+                        &index_name,
+                        &index_segments,
+                        index_metric,
+                    )
+                    .await;
             }
 
             log::trace!("index found for vector search");
@@ -3784,9 +3965,24 @@ impl Scanner {
                     "Refine factor cannot be zero".to_string(),
                 ));
             }
+            // Mask data overlay files: compute which row addresses within each segment have
+            // been updated by a newer overlay so their ANN entries may be stale.
+            // These stale rows are blocked from ANN results via the prefilter and re-scored
+            // on the targeted flat path below — only the specific stale rows, not the whole
+            // fragment, so sparse overlays incur near-zero overhead.
+            let stale_rows = self.overlay_stale_vector_rows(&index_segments)?;
+            // Build a prefilter block mask for stale rows (empty = no-op fast path).
+            let overlay_block = self.stale_rows_block_mask(&stale_rows).await?;
+
             let ann_node = match vector_type {
-                DataType::FixedSizeList(_, _) => self.ann(&q, &index_segments, filter_plan).await?,
-                DataType::List(_) => self.multivec_ann(&q, &index_segments, filter_plan).await?,
+                DataType::FixedSizeList(_, _) => {
+                    self.ann(&q, &index_segments, filter_plan, overlay_block.clone())
+                        .await?
+                }
+                DataType::List(_) => {
+                    self.multivec_ann(&q, &index_segments, filter_plan, overlay_block.clone())
+                        .await?
+                }
                 _ => unreachable!(),
             };
 
@@ -3804,7 +4000,14 @@ impl Scanner {
 
             if !self.fast_search {
                 knn_node = self
-                    .knn_combined(&q, &index_name, &index_segments, knn_node, filter_plan)
+                    .knn_combined(
+                        &q,
+                        &index_name,
+                        &index_segments,
+                        &stale_rows,
+                        knn_node,
+                        filter_plan,
+                    )
                     .await?;
             }
 
@@ -3853,11 +4056,107 @@ impl Scanner {
         }
     }
 
+    /// Whether a batch (multi-query) vector search can use the shared-scan
+    /// indexed fast path ([`new_knn_batch_exec`]) instead of running one indexed
+    /// search per query vector.
+    ///
+    /// Requires all of:
+    /// - no refine step (the batch path does not yet rerank);
+    /// - fixed nprobes (`minimum_nprobes == maximum_nprobes`) — see below;
+    /// - every segment an IVF index with a flat-style sub-index (i.e. not HNSW);
+    /// - all target fragments indexed (or `fast_search`, which ignores
+    ///   unindexed fragments).
+    ///
+    /// The fixed-nprobes requirement is a *correctness* gate, not just an
+    /// optimization. The shared-scan path searches exactly `minimum_nprobes`
+    /// partitions per query, but the single-query path is adaptive: it applies a
+    /// k-dependent `early_pruning` floor and then expands probes up to
+    /// `maximum_nprobes` (late search) when a query has fewer than `k` results.
+    /// When `minimum_nprobes == maximum_nprobes` neither adjustment can fire
+    /// (pruning is capped at the maximum, and the late-search range is empty), so
+    /// the batch result is provably identical to repeated single-query search.
+    /// With adaptive nprobes the two would diverge, so we fall back to the
+    /// per-query loop, which reuses the real adaptive search and stays exact.
+    ///
+    /// Extending the shared-scan path to adaptive nprobes (a batched early/late
+    /// search) is left as a follow-up.
+    async fn batch_index_search_supported(
+        &self,
+        index_name: &str,
+        index_segments: &[IndexMetadata],
+        q: &Query,
+    ) -> Result<bool> {
+        if matches!(q.refine_factor, Some(rf) if rf > 1) {
+            return Ok(false);
+        }
+        // Only fixed nprobes is provably equivalent to single-query search; see
+        // the method docs. Adaptive nprobes falls back to the per-query loop.
+        if q.maximum_nprobes != Some(q.minimum_nprobes) {
+            return Ok(false);
+        }
+        // Decide from the index metadata (no I/O) rather than opening the index
+        // to call `supports_batch_partition_search()`: this is a planning-time
+        // gate and the single-query path likewise avoids opening the index here.
+        // An IVF index with a flat-style sub-index (i.e. not HNSW) is exactly the
+        // set for which `supports_batch_partition_search()` is true; the exec
+        // re-checks that trait as a defensive invariant. Legacy segments without
+        // details fall back.
+        let all_ivf_flat_style = index_segments.iter().all(|index| {
+            index
+                .index_details
+                .as_ref()
+                .filter(|details| !details.value.is_empty())
+                .map(|details| {
+                    let index_type =
+                        crate::index::vector::details::derive_vector_index_type(details);
+                    index_type.starts_with("IVF_") && !index_type.contains("HNSW")
+                })
+                .unwrap_or(false)
+        });
+        if !all_ivf_flat_style {
+            return Ok(false);
+        }
+        if self.fast_search {
+            return Ok(true);
+        }
+        // The batch node only searches indexed partitions, so any unindexed
+        // target fragment would silently drop rows; fall back in that case.
+        let unindexed_fragments =
+            self.retain_target_fragments(self.dataset.unindexed_fragments(index_name).await?);
+        Ok(unindexed_fragments.is_empty())
+    }
+
     async fn batch_indexed_vector_search(
         &self,
         filter_plan: &ExprFilterPlan,
         q: &Query,
+        index_name: &str,
+        index_segments: &[IndexMetadata],
+        index_metric: MetricType,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        // Fast path: when every index segment is an IVF index with a flat-style
+        // sub-index (IVF_FLAT/PQ/SQ/RQ), search all query vectors in a single
+        // pass that reads each partition's storage once and shares the prefilter
+        // across the batch. HNSW, refine, and mixed indexed/unindexed scans fall
+        // back to the per-query loop below, which never regresses behavior.
+        if self
+            .batch_index_search_supported(index_name, index_segments, q)
+            .await?
+        {
+            let mut batch_query = q.clone();
+            batch_query.metric_type = Some(index_metric);
+            let prefilter_source = self
+                .prefilter_source(filter_plan, self.get_indexed_frags(index_segments))
+                .await?;
+            return new_knn_batch_exec(
+                self.dataset.clone(),
+                index_segments,
+                &batch_query,
+                self.nearest_query_count,
+                prefilter_source,
+            );
+        }
+
         let query_dim = q.key.len() / self.nearest_query_count;
         let mut query_plans = Vec::with_capacity(self.nearest_query_count);
 
@@ -3939,6 +4238,7 @@ impl Scanner {
         q: &Query,
         index_name: &str,
         indexed_segments: &[IndexMetadata],
+        stale_rows: &HashMap<u32, RoaringBitmap>,
         mut knn_node: Arc<dyn ExecutionPlan>,
         filter_plan: &ExprFilterPlan,
     ) -> Result<Arc<dyn ExecutionPlan>> {
@@ -3955,30 +4255,41 @@ impl Scanner {
             self.dataset.unindexed_fragments(index_name).await?
         };
 
-        if !fallback_fragments.is_empty() {
-            let q = q.clone();
-            debug_assert!(q.metric_type.is_some());
+        let has_fallback = !fallback_fragments.is_empty();
+        let has_stale = !stale_rows.is_empty();
 
-            // If the vector column is not present, we need to take the vector column, so
-            // that the distance value is comparable with the flat search ones.
-            if knn_node.schema().column_with_name(&q.column).is_none() {
-                let vector_projection = self
-                    .dataset
-                    .empty_projection()
-                    .union_column(&q.column, OnMissing::Error)
-                    .unwrap();
-                knn_node = self.take(knn_node, vector_projection)?;
-            }
+        if !has_fallback && !has_stale {
+            return Ok(knn_node);
+        }
 
-            let mut columns = vec![q.column.clone()];
-            if let Some(expr) = filter_plan.full_expr.as_ref() {
-                let filter_columns = Planner::column_names_in_expr(expr);
-                columns.extend(filter_columns);
-            }
-            let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns).unwrap());
-            // Note: we could try and use the scalar indices here to reduce the scope of this scan but the
-            // most common case is that fragments that are newer than the vector index are going to be newer
-            // than the scalar indices anyways
+        let q = q.clone();
+        debug_assert!(q.metric_type.is_some());
+
+        // Ensure the vector column is present for distance computation.
+        if knn_node.schema().column_with_name(&q.column).is_none() {
+            let vector_projection = self
+                .dataset
+                .empty_projection()
+                .union_column(&q.column, OnMissing::Error)?;
+            knn_node = self.take(knn_node, vector_projection)?;
+        }
+
+        let mut columns = vec![q.column.clone()];
+        if let Some(expr) = filter_plan.full_expr.as_ref() {
+            let filter_columns = Planner::column_names_in_expr(expr);
+            columns.extend(filter_columns);
+        }
+
+        // Collect flat-path plans; union order matches original (flat before ANN) so test snapshots
+        // and downstream plan analyses remain stable.
+        let mut flat_inputs: Vec<Arc<dyn ExecutionPlan>> = Vec::new();
+
+        // Flat KNN for unindexed (new-data) fragments.
+        if has_fallback {
+            let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns)?);
+            // Note: we could try and use the scalar indices here to reduce the scope of this scan
+            // but the most common case is that fragments newer than the vector index are also
+            // newer than the scalar indices.
             let mut scan_node = self.scan_fragments(
                 true,
                 false,
@@ -3989,39 +4300,51 @@ impl Scanner {
                 Arc::new(fallback_fragments),
                 // Can't pushdown limit/offset in an ANN search
                 None,
-                // We are re-ordering anyways, so no need to get data in data
-                // in a deterministic order.
+                // We are re-ordering anyways, so no need to get data in a deterministic order.
                 false,
             );
-
             if let Some(expr) = filter_plan.full_expr.as_ref() {
-                // If there is a prefilter we need to manually apply it to the new data
                 scan_node = Arc::new(LanceFilterExec::try_new(expr.clone(), scan_node)?);
             }
-            // first we do flat search on just the new data
-            let topk_appended = self.flat_knn(scan_node, &q)?;
-
-            // To do a union, we need to make the schemas match. Right now
-            // knn_node: _distance, _rowid, vector
-            // topk_appended: vector, <filter columns?>, _rowid, _distance
-            let topk_appended = project(topk_appended, knn_node.schema().as_ref())?;
-            assert!(
-                topk_appended
-                    .schema()
-                    .equivalent_names_and_types(&knn_node.schema())
-            );
-            // union
-            let unioned = UnionExec::try_new(vec![Arc::new(topk_appended), knn_node])?;
-            // Enforce only 1 partition.
-            let unioned = RepartitionExec::try_new(
-                unioned,
-                datafusion::physical_plan::Partitioning::RoundRobinBatch(1),
-            )?;
-            // then we do a flat search on KNN(new data) + ANN(indexed data)
-            return self.flat_knn(Arc::new(unioned), &q);
+            let topk_fallback = self.flat_knn(scan_node, &q)?;
+            let topk_fallback: Arc<dyn ExecutionPlan> =
+                Arc::new(project(topk_fallback, knn_node.schema().as_ref())?);
+            flat_inputs.push(topk_fallback);
+        }
+
+        // Flat KNN for stale rows only (row-level precision).
+        // Only specific row addresses need re-scoring, not the whole fragment, so sparse overlays
+        // incur near-zero overhead.
+        if has_stale {
+            // Fetch vector + filter columns for the stale rows. `flat_knn` sorts by row id, so the
+            // take must carry it (the fallback scan above gets it via `scan_fragments`).
+            let mut take_proj = self
+                .dataset
+                .empty_projection()
+                .with_row_id()
+                .union_column(&q.column, OnMissing::Error)?;
+            if let Some(expr) = filter_plan.full_expr.as_ref() {
+                let filter_columns = Planner::column_names_in_expr(expr);
+                take_proj = take_proj.union_columns(filter_columns, OnMissing::Error)?;
+            }
+            let mut stale_node = self.stale_rows_take(stale_rows, take_proj).await?;
+            if let Some(expr) = filter_plan.full_expr.as_ref() {
+                stale_node = Arc::new(LanceFilterExec::try_new(expr.clone(), stale_node)?);
+            }
+            let topk_stale = self.flat_knn(stale_node, &q)?;
+            let topk_stale: Arc<dyn ExecutionPlan> =
+                Arc::new(project(topk_stale, knn_node.schema().as_ref())?);
+            flat_inputs.push(topk_stale);
         }
 
-        Ok(knn_node)
+        // Union: flat paths first (matching original order), then ANN results.
+        flat_inputs.push(knn_node);
+        let unioned = UnionExec::try_new(flat_inputs)?;
+        let unioned = RepartitionExec::try_new(
+            unioned,
+            datafusion::physical_plan::Partitioning::RoundRobinBatch(1),
+        )?;
+        self.flat_knn(Arc::new(unioned), &q)
     }
 
     #[async_recursion]
@@ -4052,29 +4375,252 @@ impl Scanner {
         }
     }
 
-    /// Given an index query, split the fragments into two sets
+    /// Given an index query, split the fragments into two groups and collect per-row stale data.
     ///
-    /// The first set is the relevant fragments, which are covered by ALL indices in the query
-    /// The second set is the missing fragments, which are missed by at least one index
+    /// - `relevant_frags`: covered by ALL indices. Stale rows within them are returned separately
+    ///   so callers can block them from `MaterializeIndexExec` and re-score via a targeted take.
+    /// - `missing_frags`: not covered by at least one index; fall back to full scan + filter.
+    /// - `stale_rows`: per-fragment row offsets whose indexed values are stale due to a data
+    ///   overlay committed after the index was built (field-aware, version-gated). Empty when no
+    ///   overlays are present.
     ///
-    /// There is no point in handling the case where a fragment is covered by some (but not all)
-    /// of the indices.  If we have to do a full scan of the fragment then we do it
+    /// There is no point in partially indexing a fragment (some indices cover it, others do not).
+    /// If we have to do a full scan of a fragment for any reason, we do it entirely.
     async fn partition_frags_by_coverage(
         &self,
         index_expr: &ScalarIndexExpr,
         fragments: Arc<Vec<Fragment>>,
-    ) -> Result<(Vec<Fragment>, Vec<Fragment>)> {
+    ) -> Result<(Vec<Fragment>, Vec<Fragment>, HashMap<u32, RoaringBitmap>)> {
         let covered_frags = self.fragments_covered_by_index_query(index_expr).await?;
+        let stale_rows = self
+            .overlay_stale_index_rows(index_expr, &fragments)
+            .await?;
         let mut relevant_frags = Vec::with_capacity(fragments.len());
         let mut missing_frags = Vec::with_capacity(fragments.len());
         for fragment in fragments.iter() {
             if covered_frags.contains(fragment.id as u32) {
+                // Indexed fragments stay on the indexed path. Stale rows within them are blocked
+                // from the index result and re-evaluated separately via a targeted take.
                 relevant_frags.push(fragment.clone());
             } else {
                 missing_frags.push(fragment.clone());
             }
         }
-        Ok((relevant_frags, missing_frags))
+        Ok((relevant_frags, missing_frags, stale_rows))
+    }
+
+    /// Per-row stale offsets for each fragment whose indexed values may be stale because an
+    /// overlay committed *after* an index was built touches a field that index covers.
+    ///
+    /// The check is field-aware (an overlay touching only unindexed fields excludes nothing) and
+    /// version-gated (an overlay with `committed_version <= index.dataset_version` is already
+    /// incorporated by the index), via [`overlay_exclusion_offsets`].
+    async fn overlay_stale_index_rows(
+        &self,
+        index_expr: &ScalarIndexExpr,
+        fragments: &[Fragment],
+    ) -> Result<HashMap<u32, RoaringBitmap>> {
+        // Overlays are rare; skip all index loading when none of the candidate fragments has one.
+        let overlaid_frags = overlaid_fragments(fragments);
+        if overlaid_frags.is_empty() {
+            return Ok(HashMap::new());
+        }
+
+        // Walk the (boolean) index expression tree to collect leaf searches.
+        let mut searches = Vec::new();
+        let mut stack = vec![index_expr];
+        while let Some(expr) = stack.pop() {
+            match expr {
+                ScalarIndexExpr::Not(inner) => stack.push(inner),
+                ScalarIndexExpr::And(lhs, rhs) | ScalarIndexExpr::Or(lhs, rhs) => {
+                    stack.push(lhs);
+                    stack.push(rhs);
+                }
+                ScalarIndexExpr::Query(search) => searches.push(search),
+            }
+        }
+
+        // `load_named_scalar_segments` returns cached index metadata — no disk I/O on the hot
+        // path. Even without the cache, this code is only reached when at least one fragment has
+        // overlays (rare), so the per-leaf cost is acceptable.
+        let mut stale: HashMap<u32, RoaringBitmap> = HashMap::new();
+        for search in searches {
+            let segments = load_named_scalar_segments(
+                self.dataset.as_ref(),
+                &search.column,
+                &search.index_name,
+            )
+            .await?;
+            for segment in &segments {
+                collect_overlay_stale_rows_for_segment(
+                    segment,
+                    &overlaid_frags,
+                    &mut stale,
+                    self.dataset.schema(),
+                )?;
+            }
+        }
+        Ok(stale)
+    }
+
+    /// Compute per-row stale data for a vector index's segments.
+    ///
+    /// Returns a map from fragment_id to the set of row offsets within that fragment that are stale
+    /// (their vector values have been updated by a newer overlay since the index was built). An
+    /// empty map means no stale rows — the fast path where no masking is needed.
+    fn overlay_stale_vector_rows(
+        &self,
+        segments: &[IndexMetadata],
+    ) -> Result<HashMap<u32, RoaringBitmap>> {
+        // Scope to the query's target fragments (all dataset fragments if unscoped).
+        let dataset_frags = self.dataset.fragments();
+        let fragments: &[Fragment] = match self.fragments.as_ref() {
+            Some(f) => f.as_slice(),
+            None => dataset_frags.as_slice(),
+        };
+        let overlaid_frags = overlaid_fragments(fragments);
+        if overlaid_frags.is_empty() {
+            return Ok(HashMap::new());
+        }
+        let mut stale: HashMap<u32, RoaringBitmap> = HashMap::new();
+        for segment in segments {
+            collect_overlay_stale_rows_for_segment(
+                segment,
+                &overlaid_frags,
+                &mut stale,
+                self.dataset.schema(),
+            )?;
+        }
+        Ok(stale)
+    }
+
+    /// Compute which FTS segments are stale due to data overlay files committed after the
+    /// index was built, and which fragments must therefore fall back to the flat text path.
+    ///
+    /// Returns `(flat_frag_ids, Some(fresh_segments))` when overlays are present:
+    /// - `flat_frag_ids`: fragment IDs that must be scanned flat (stale fragments, plus any
+    ///   other fragments co-located in a segment that covers a stale one — the whole segment is
+    ///   excluded, so all fragments it covered must move to flat).
+    /// - `fresh_segments`: the subset of FTS segments that cover no stale fragment; safe to
+    ///   pass to `MatchQueryExec::new_with_segments`.
+    ///
+    /// Returns `(empty, None)` on the fast path (no overlays, or no segments load).
+    async fn fts_stale_frags_and_fresh_segments(
+        &self,
+        column: &str,
+        target_fragments: &[Fragment],
+    ) -> Result<(RoaringBitmap, Option<Vec<IndexMetadata>>)> {
+        // Fast path: no overlays on any target fragment.
+        if target_fragments.iter().all(|f| f.overlays.is_empty()) {
+            return Ok((RoaringBitmap::new(), None));
+        }
+
+        let Some(segments) = load_segments(&self.dataset, column).await? else {
+            return Ok((RoaringBitmap::new(), None));
+        };
+
+        let overlaid_frags = overlaid_fragments(target_fragments);
+        let mut stale_frag_ids = RoaringBitmap::new();
+        for seg in &segments {
+            collect_overlay_stale_frags(
+                seg,
+                &overlaid_frags,
+                &mut stale_frag_ids,
+                self.dataset.schema(),
+            )?;
+        }
+
+        if stale_frag_ids.is_empty() {
+            // Overlays exist but none are on this FTS column or predate the index.
+            return Ok((stale_frag_ids, None));
+        }
+
+        // Any segment covering a stale fragment is excluded from the indexed path.
+        // All fragments covered by that segment (stale + co-located fresh ones) must
+        // fall to the flat path, since the indexed path no longer covers them.
+        let mut flat_frag_ids = stale_frag_ids.clone();
+        let mut fresh_segments = Vec::with_capacity(segments.len());
+        for seg in segments {
+            match &seg.fragment_bitmap {
+                Some(bm) if !bm.is_disjoint(&stale_frag_ids) => {
+                    flat_frag_ids |= bm;
+                    // exclude this segment from the indexed path
+                }
+                Some(_) => fresh_segments.push(seg),
+                None => {
+                    // Coverage unknown (legacy segment without a fragment bitmap): we can neither
+                    // trust it to exclude overlay-stale rows nor tell which fragments it indexes.
+                    // Exclude it from the indexed path and route every target fragment to flat.
+                    flat_frag_ids.extend(target_fragments.iter().map(|f| f.id as u32));
+                }
+            }
+        }
+
+        Ok((flat_frag_ids, Some(fresh_segments)))
+    }
+
+    /// Collect the stale rows into a [`RowAddrTreeMap`] in the domain the index results use.
+    ///
+    /// Index results are in the row-id domain (see `ScalarQuery::evaluate_nullable`), and a
+    /// physical row address equals its row id only when the dataset does not use stable row ids.
+    /// Under stable row ids the addresses are translated to their row ids so the result lines up
+    /// with the index output it is combined with (block mask) or taken against.
+    async fn stale_rows_in_id_domain(
+        &self,
+        stale_rows: &HashMap<u32, RoaringBitmap>,
+    ) -> Result<RowAddrTreeMap> {
+        let mut tree_map = RowAddrTreeMap::new();
+        for (&frag_id, offsets) in stale_rows {
+            tree_map.insert_bitmap(frag_id, offsets.clone());
+        }
+        if self.dataset.manifest.uses_stable_row_ids() {
+            tree_map = translate_addr_treemap_to_row_ids(&self.dataset, &tree_map).await?;
+        }
+        Ok(tree_map)
+    }
+
+    /// Build a block-list mask over stale rows, or `None` when there are none.
+    ///
+    /// The mask removes these rows from an index result so the index never emits them; they
+    /// are re-evaluated against their current (overlay-merged) values on a targeted take path
+    /// (see [`Self::stale_rows_take`]).
+    async fn stale_rows_block_mask(
+        &self,
+        stale_rows: &HashMap<u32, RoaringBitmap>,
+    ) -> Result<Option<RowAddrMask>> {
+        if stale_rows.is_empty() {
+            return Ok(None);
+        }
+        let tree_map = self.stale_rows_in_id_domain(stale_rows).await?;
+        Ok(Some(RowAddrMask::from_block(tree_map)))
+    }
+
+    /// Take the stale rows by physical address, projecting `projection`, to re-evaluate only
+    /// those rows (rather than their whole fragments) against their current overlay-merged values.
+    ///
+    /// The rows are identified by an address allow list routed through `FilteredReadExec`, not a
+    /// `_rowid` column (`_rowid` and row address diverge under stable row ids — see
+    /// [`Self::stale_rows_in_id_domain`]).
+    async fn stale_rows_take(
+        &self,
+        stale_rows: &HashMap<u32, RoaringBitmap>,
+        projection: Projection,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let take_id_map = self.stale_rows_in_id_domain(stale_rows).await?;
+        let take_ids: Vec<u64> = take_id_map
+            .row_addrs()
+            .map(|it| it.map(u64::from).collect())
+            .unwrap_or_default();
+        let index_input = self.u64s_as_take_input(take_ids)?;
+        let mut read_options = FilteredReadOptions::new(projection);
+        if let Some(fragments) = self.fragments.as_ref() {
+            read_options = read_options.with_fragments(Arc::new(fragments.clone()));
+        }
+        Ok(Arc::new(FilteredReadExec::try_new(
+            self.dataset.clone(),
+            read_options,
+            Some(index_input),
+        )?))
     }
 
     // First perform a lookup in a scalar index for ids and then perform a take on the
@@ -4096,16 +4642,24 @@ impl Scanner {
 
         let needs_recheck = index_expr.needs_recheck();
 
-        // Figure out which fragments are covered by ALL indices
-        let (relevant_frags, missing_frags) = self
+        // Figure out which fragments are covered by ALL indices, and which rows within
+        // covered fragments are stale due to data overlay files.
+        let (relevant_frags, missing_frags, stale_rows) = self
             .partition_frags_by_coverage(index_expr, fragments)
             .await?;
 
-        let mut plan: Arc<dyn ExecutionPlan> = Arc::new(MaterializeIndexExec::new(
+        // Build the MaterializeIndexExec, blocking stale row addresses so the index never
+        // emits them. Stale rows are re-scored separately via a targeted take below.
+        let mat_exec = MaterializeIndexExec::new(
             self.dataset.clone(),
             index_expr.clone(),
             Arc::new(relevant_frags),
-        ));
+        );
+        let mat_exec = match self.stale_rows_block_mask(&stale_rows).await? {
+            Some(block) => mat_exec.with_overlay_block(block),
+            None => mat_exec,
+        };
+        let mut plan: Arc<dyn ExecutionPlan> = Arc::new(mat_exec);
 
         let refine_expr = filter_plan.refine_expr.as_ref();
 
@@ -4151,6 +4705,21 @@ impl Scanner {
             plan = Arc::new(AddRowAddrExec::try_new(plan, self.dataset.clone(), 0)?);
         }
 
+        // Both the missing-fragments path (full scan) and the stale-rows path (targeted take)
+        // need the user's projection extended with any filter columns. Compute it once.
+        let fallback_projection: Option<Projection> =
+            if !missing_frags.is_empty() || !stale_rows.is_empty() {
+                let filter = filter_plan.full_expr.as_ref().expect_ok()?;
+                let filter_cols = Planner::column_names_in_expr(filter);
+                Some(
+                    projection
+                        .clone()
+                        .union_columns(filter_cols, OnMissing::Error)?,
+                )
+            } else {
+                None
+            };
+
         let new_data_path: Option<Arc<dyn ExecutionPlan>> = if !missing_frags.is_empty() {
             log::trace!(
                 "scalar_indexed_scan will need full scan of {} missing fragments",
@@ -4169,10 +4738,8 @@ impl Scanner {
             // If there were no extra columns then we still need the project
             // because Materialize -> Take puts the row id at the left and
             // Scan puts the row id at the right
-            let filter = filter_plan.full_expr.as_ref().unwrap();
-            let filter_cols = Planner::column_names_in_expr(filter);
-            let scan_projection = projection.union_columns(filter_cols, OnMissing::Error)?;
-
+            let scan_projection = fallback_projection.clone().expect_ok()?;
+            let filter = filter_plan.full_expr.as_ref().expect_ok()?;
             let scan_schema = Arc::new(scan_projection.to_bare_schema());
             let scan_arrow_schema = Arc::new(scan_schema.as_ref().into());
             let planner = Planner::new(scan_arrow_schema);
@@ -4201,16 +4768,37 @@ impl Scanner {
             None
         };
 
-        if let Some(new_data_path) = new_data_path {
-            let unioned = UnionExec::try_new(vec![plan, new_data_path])?;
-            // Enforce only 1 partition.
-            let unioned = Arc::new(RepartitionExec::try_new(
-                unioned,
-                datafusion::physical_plan::Partitioning::RoundRobinBatch(1),
-            )?);
-            Ok(unioned)
+        // Stale-Take path: re-evaluate only the stale row addresses against the full filter
+        // (row-level optimization). These rows were blocked from the index result above;
+        // here we take their current (overlay-merged) values and re-apply the predicate.
+        // The schema matches `plan` via `project(…, plan.schema())`.
+        let stale_take_path: Option<Arc<dyn ExecutionPlan>> = if stale_rows.is_empty() {
+            None
         } else {
+            let filter = filter_plan.full_expr.as_ref().expect_ok()?;
+            let take_projection = fallback_projection.expect_ok()?;
+
+            let stale_node = self.stale_rows_take(&stale_rows, take_projection).await?;
+
+            let planner = Planner::new(stale_node.schema());
+            let optimized_filter = planner.optimize_expr(filter.clone())?;
+            let filtered = Arc::new(LanceFilterExec::try_new(optimized_filter, stale_node)?);
+            Some(Arc::new(project(filtered, plan.schema().as_ref())?))
+        };
+
+        let extra_paths: Vec<Arc<dyn ExecutionPlan>> = [new_data_path, stale_take_path]
+            .into_iter()
+            .flatten()
+            .collect();
+        if extra_paths.is_empty() {
             Ok(plan)
+        } else {
+            let all_paths = std::iter::once(plan).chain(extra_paths).collect();
+            let unioned = UnionExec::try_new(all_paths)?;
+            Ok(Arc::new(RepartitionExec::try_new(
+                unioned,
+                datafusion::physical_plan::Partitioning::RoundRobinBatch(1),
+            )?))
         }
     }
 
@@ -4360,7 +4948,8 @@ impl Scanner {
                         .dataset
                         .empty_projection()
                         .union_column(&column, OnMissing::Error)?;
-                    self.take(input, projection)?
+                    let input = self.take(input, projection)?;
+                    self.ensure_column_alias(input, &column)?
                 } else {
                     input
                 };
@@ -4410,7 +4999,8 @@ impl Scanner {
                         .dataset
                         .empty_projection()
                         .union_column(&column, OnMissing::Error)?;
-                    self.take(input, projection)?
+                    let input = self.take(input, projection)?;
+                    self.ensure_column_alias(input, &column)?
                 } else {
                     input
                 };
@@ -4642,11 +5232,18 @@ impl Scanner {
         q: &Query,
         index: &[IndexMetadata],
         filter_plan: &ExprFilterPlan,
+        overlay_block: Option<RowAddrMask>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let prefilter_source = self
             .prefilter_source(filter_plan, self.get_indexed_frags(index))
             .await?;
-        let inner_fanout_search = new_knn_exec(self.dataset.clone(), index, q, prefilter_source)?;
+        let inner_fanout_search = new_knn_exec(
+            self.dataset.clone(),
+            index,
+            q,
+            prefilter_source,
+            overlay_block,
+        )?;
         let sort_expr = PhysicalSortExpr {
             expr: expressions::col(DIST_COL, inner_fanout_search.schema().as_ref())?,
             options: SortOptions {
@@ -4673,6 +5270,7 @@ impl Scanner {
         q: &Query,
         index: &[IndexMetadata],
         filter_plan: &ExprFilterPlan,
+        overlay_block: Option<RowAddrMask>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         // we split the query procedure into two steps:
         // 1. collect the candidates by vector searching on each query vector
@@ -4705,6 +5303,7 @@ impl Scanner {
                 index,
                 &query,
                 prefilter_source.clone(),
+                overlay_block.clone(),
             )?;
             let sort_expr = PhysicalSortExpr {
                 expr: expressions::col(DIST_COL, ann_node.schema().as_ref())?,
@@ -4785,11 +5384,14 @@ impl Scanner {
         // are not in the fragments we are scanning.
         if filter_plan.is_exact_index_search() && self.fragments.is_none() {
             let index_query = filter_plan.index_query.as_ref().expect_ok()?;
-            let (_, missing_frags) = self
+            let (_, missing_frags, stale_rows) = self
                 .partition_frags_by_coverage(index_query, fragments.clone())
                 .await?;
 
-            if missing_frags.is_empty() || self.fast_search {
+            // Overlay-stale rows must never reach the direct ScalarIndexExec path: it would hand
+            // ANN/FTS a selection vector containing rows whose indexed values are now stale. When
+            // any exist, fall through to the filtered-read prefilter, which masks them.
+            if stale_rows.is_empty() && (missing_frags.is_empty() || self.fast_search) {
                 log::trace!("prefilter entirely satisfied by exact index search");
                 let result_format = self.index_expr_result_format();
                 // We can only avoid materializing the index for a prefilter if:
@@ -4824,12 +5426,57 @@ impl Scanner {
     }
 
     /// Take row indices produced by input plan from the dataset (with projection)
+    ///
+    /// Planned as a [`FilteredReadExec`] row-stream read; legacy (v1) storage
+    /// keeps using [`TakeExec`].
     #[allow(deprecated)]
     fn take(
         &self,
         input: Arc<dyn ExecutionPlan>,
         output_projection: Projection,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        let fields_to_take = output_projection
+            .clone()
+            .subtract_arrow_schema(input.schema().as_ref(), OnMissing::Ignore)?;
+        if !fields_to_take.has_data_fields()
+            && !fields_to_take.with_row_id
+            && !fields_to_take.with_row_addr
+        {
+            // No new columns needed
+            return Ok(input);
+        }
+
+        let input_schema = input.schema();
+        let has_row_id = input_schema.column_with_name(ROW_ID).is_some();
+        let has_row_addr = input_schema.column_with_name(ROW_ADDR).is_some();
+        // The v1 reader cannot serve a FilteredReadExec
+        if !self.dataset.is_legacy_storage() && (has_row_id || has_row_addr) {
+            // Pass the full (un-subtracted) target so a rebuild against a
+            // different child re-derives what to fetch, and preserve carried
+            // identity columns (downstream nodes may key off them; the final
+            // ProjectionExec trims for free)
+            let mut projection = output_projection;
+            projection.with_row_id |= has_row_id;
+            projection.with_row_addr |= has_row_addr;
+            let mut read_options = FilteredReadOptions::new(projection);
+            if self.include_deleted_rows {
+                // Forwarded so the row-stream read rejects it: deleted rows
+                // carry a null row id, which the take would silently drop
+                read_options = read_options.with_deleted_rows()?;
+            }
+            if let Some(batch_size) = self.batch_size {
+                read_options = read_options.with_batch_size(batch_size as u32);
+            }
+            if let Some(fragments) = &self.fragments {
+                read_options = read_options.with_fragments(Arc::new(fragments.clone()));
+            }
+            return Ok(Arc::new(FilteredReadExec::try_new(
+                self.dataset.clone(),
+                read_options,
+                Some(input),
+            )?));
+        }
+
         let coalesced = Arc::new(CoalesceBatchesExec::new(
             input.clone(),
             self.get_batch_size(),
@@ -4868,7 +5515,12 @@ impl Scanner {
 
     #[instrument(level = "info", skip(self))]
     pub async fn explain_plan(&self, verbose: bool) -> Result<String> {
-        let plan = self.create_plan().await?;
+        // Box the plan-building future at the call site: `create_plan`'s inlined async
+        // layout otherwise exceeds rustc's depth limit here. It has to be boxed at the
+        // call site rather than inside `create_plan` — boxing internally turns the
+        // future's `Send` check into a `Box<Future>: Send` trait obligation that
+        // overflows the solver through the cache types (E0275 in downstream crates).
+        let plan = Box::pin(self.create_plan()).await?;
         let display = DisplayableExecutionPlan::new(plan.as_ref());
 
         Ok(format!("{}", display.indent(verbose)))
@@ -4895,24 +5547,25 @@ impl Scanner {
     }
 }
 
+fn is_fts_indexable_field(field: &Field) -> bool {
+    match field.data_type() {
+        DataType::Utf8 | DataType::LargeUtf8 => true,
+        DataType::List(inner_field) | DataType::LargeList(inner_field) => {
+            matches!(
+                inner_field.data_type(),
+                DataType::Utf8 | DataType::LargeUtf8
+            )
+        }
+        _ => false,
+    }
+}
+
 // Search over all indexed fields including nested ones, collecting columns that have an
 // inverted index
 async fn fts_indexed_columns(dataset: Arc<Dataset>) -> Result<Vec<String>> {
     let mut indexed_columns = Vec::new();
     for field in dataset.schema().fields_pre_order() {
-        // Check if this field is a string type that could have an inverted index
-        let is_string_field = match field.data_type() {
-            DataType::Utf8 | DataType::LargeUtf8 => true,
-            DataType::List(inner_field) | DataType::LargeList(inner_field) => {
-                matches!(
-                    inner_field.data_type(),
-                    DataType::Utf8 | DataType::LargeUtf8
-                )
-            }
-            _ => false,
-        };
-
-        if is_string_field {
+        if is_fts_indexable_field(field) {
             // Build the full field path for nested fields
             let column_path =
                 if let Some(ancestors) = dataset.schema().field_ancestry_by_id(field.id) {
@@ -5009,8 +5662,10 @@ pub mod test_dataset {
         IndexType,
         scalar::{ScalarIndexParams, inverted::tokenizer::InvertedIndexParams},
         vector::{
+            hnsw::builder::HnswBuildParams,
             ivf::IvfBuildParams,
             kmeans::{KMeansParams, train_kmeans},
+            sq::builder::SQBuildParams,
         },
     };
     use lance_linalg::distance::DistanceType;
@@ -5112,8 +5767,31 @@ pub mod test_dataset {
         }
 
         pub async fn make_vector_index(&mut self) -> Result<()> {
-            let params = VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2);
-            self.dataset
+            self.make_vector_index_with_metric(MetricType::L2).await
+        }
+
+        pub async fn make_vector_index_with_metric(&mut self, metric: MetricType) -> Result<()> {
+            let params = VectorIndexParams::ivf_pq(2, 8, 2, metric, 2);
+            self.dataset
+                .create_index(
+                    &["vec"],
+                    IndexType::Vector,
+                    Some("idx".to_string()),
+                    &params,
+                    true,
+                )
+                .await?;
+            Ok(())
+        }
+
+        pub async fn make_ivf_hnsw_index(&mut self) -> Result<()> {
+            let params = VectorIndexParams::with_ivf_hnsw_sq_params(
+                MetricType::L2,
+                IvfBuildParams::new(2),
+                HnswBuildParams::default(),
+                SQBuildParams::default(),
+            );
+            self.dataset
                 .create_index(
                     &["vec"],
                     IndexType::Vector,
@@ -5196,7 +5874,13 @@ pub mod test_dataset {
         }
 
         pub async fn make_fts_index(&mut self) -> Result<()> {
-            let params = InvertedIndexParams::default().with_position(true);
+            // These scanner tests search for the token "s" (from the `s-{N}`
+            // column values) to exercise fragment/append coverage, and "s" is
+            // in the full English stop-word list. Keep the token searchable;
+            // stop-word behavior itself is covered by the tokenizer tests.
+            let params = InvertedIndexParams::default()
+                .with_position(true)
+                .remove_stop_words(false);
             self.dataset
                 .create_index(&["s"], IndexType::Inverted, None, &params, true)
                 .await?;
@@ -5628,6 +6312,149 @@ mod test {
         Ok(())
     }
 
+    // Regression for #6580: a scan with `filter` + `project` of a
+    // `(Large)List<Struct>` column used to panic in `merge_with_schema`
+    // (called from `TakeStream::map_batch`) because the filtered batch arrived
+    // as a sliced view of a larger batch and the cloned list offsets did not
+    // start at zero. The trigger requires (a) a `(Large)List<Struct>`
+    // projection where the struct is split across `filtered_read` and
+    // `TakeExec` and (b) a sparse-tail selectivity pattern so the trailing
+    // filter result lands deep inside the values buffer of its source batch.
+    // Parametrized over `List`/`LargeList` since the fix touches both offset
+    // widths in `merge_with_schema`.
+    #[rstest]
+    #[tokio::test]
+    async fn test_filter_project_list_struct_sparse_tail(
+        // The panic is specific to v2.x storage; the legacy reader takes a
+        // different code path. V2_0 and V2_2 are the versions called out in
+        // the original report.
+        #[values(
+            LanceFileVersion::V2_0,
+            LanceFileVersion::Stable,
+            LanceFileVersion::V2_2
+        )]
+        data_storage_version: LanceFileVersion,
+        #[values(false, true)] large_list: bool,
+    ) {
+        use arrow_array::{LargeListArray, ListArray, UInt16Array};
+        use arrow_buffer::{OffsetBuffer, ScalarBuffer};
+
+        let struct_fields = Fields::from(vec![
+            Arc::new(ArrowField::new("a", DataType::Int32, true)),
+            Arc::new(ArrowField::new("b", DataType::Int32, true)),
+        ]);
+        let item_field = Arc::new(ArrowField::new(
+            "item",
+            DataType::Struct(struct_fields.clone()),
+            true,
+        ));
+        let items_dtype = if large_list {
+            DataType::LargeList(item_field.clone())
+        } else {
+            DataType::List(item_field.clone())
+        };
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("grp", DataType::UInt16, false),
+            ArrowField::new("items", items_dtype, false),
+        ]));
+
+        let make_batch = |start: i32, n: usize, group: u16| -> RecordBatch {
+            let ids = Int32Array::from_iter_values(start..start + n as i32);
+            let groups = UInt16Array::from(vec![group; n]);
+
+            let mut offsets = Vec::with_capacity(n + 1);
+            let mut a_vals: Vec<i32> = Vec::new();
+            let mut b_vals: Vec<i32> = Vec::new();
+            offsets.push(0i64);
+            for i in 0..n {
+                // Variable-length lists (1..=18) so offsets don't land on
+                // batch-row boundaries.
+                let len = 1 + (i % 18);
+                for j in 0..len {
+                    a_vals.push(j as i32);
+                    b_vals.push(-(j as i32));
+                }
+                offsets.push(a_vals.len() as i64);
+            }
+            let struct_arr = Arc::new(StructArray::new(
+                struct_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from(a_vals)) as ArrayRef,
+                    Arc::new(Int32Array::from(b_vals)) as ArrayRef,
+                ],
+                None,
+            ));
+            let items: ArrayRef = if large_list {
+                Arc::new(LargeListArray::new(
+                    item_field.clone(),
+                    OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                    struct_arr,
+                    None,
+                ))
+            } else {
+                let offsets_i32: Vec<i32> = offsets.iter().map(|&o| o as i32).collect();
+                Arc::new(ListArray::new(
+                    item_field.clone(),
+                    OffsetBuffer::new(ScalarBuffer::from(offsets_i32)),
+                    struct_arr,
+                    None,
+                ))
+            };
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(ids) as ArrayRef,
+                    Arc::new(groups) as ArrayRef,
+                    items,
+                ],
+            )
+            .unwrap()
+        };
+
+        // Sparse-tail selectivity (matching the original report's shape at a
+        // smaller scale): a large leading block of matches, a large gap of
+        // non-matches, then a small trailing match. Single fragment.
+        let batches = vec![
+            make_batch(0, 100_000, 7),
+            make_batch(100_000, 400_000, 1),
+            make_batch(500_000, 7_300, 7),
+        ];
+
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write(
+            reader,
+            "memory://",
+            Some(WriteParams {
+                max_rows_per_file: 1_000_000,
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Force a column split inside the `items` struct by marking `items.b`
+        // as a late-materialized field: `filtered_read` returns the batch with
+        // `items.a`, and `TakeExec` adds `items.b`. `merge_with_schema` then
+        // takes its `List<Struct>` branch, which is where the panic was.
+        let items_b_field_id = dataset
+            .schema()
+            .field("items")
+            .unwrap()
+            .child("item")
+            .unwrap()
+            .child("b")
+            .unwrap()
+            .id as u32;
+        let mut scan = dataset.scan();
+        scan.filter("grp = 7").unwrap();
+        scan.project(&["id", "items"]).unwrap();
+        scan.materialization_style(MaterializationStyle::AllEarlyExcept(vec![items_b_field_id]));
+        let result = scan.try_into_batch().await.unwrap();
+        assert_eq!(result.num_rows(), 107_300);
+    }
+
     #[tokio::test]
     async fn test_scan_regexp_match_and_non_empty_captions() {
         // Build a small dataset with three Utf8 columns and verify the full
@@ -6079,6 +6906,7 @@ mod test {
         k: usize,
         use_index: bool,
         distance_range: Option<(Option<f32>, Option<f32>)>,
+        nprobes: Option<usize>,
     ) {
         let query_count = query_values.len() / 32;
         assert_eq!(batch.num_rows(), query_count * k);
@@ -6089,6 +6917,12 @@ mod test {
             let mut scan = dataset.scan();
             scan.nearest("vec", &query, k).unwrap();
             scan.use_index(use_index);
+            // Pin nprobes to match the batch query: the single-query indexed path
+            // otherwise adaptively expands nprobes, which would make equivalence
+            // depend on data distribution rather than be guaranteed.
+            if let Some(nprobes) = nprobes {
+                scan.nprobes(nprobes);
+            }
             if let Some((lower, upper)) = distance_range {
                 scan.distance_range(lower, upper);
             }
@@ -6167,7 +7001,8 @@ mod test {
                 "query_index {query_index} should have exactly {k} rows"
             );
         }
-        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await;
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None, None)
+            .await;
 
         let mut scan_with_vec = dataset.scan();
         scan_with_vec.nearest("vec", &queries, k).unwrap();
@@ -6185,6 +7020,7 @@ mod test {
             k,
             false,
             None,
+            None,
         )
         .await;
 
@@ -6238,7 +7074,8 @@ mod test {
         assert_query_index_field(&batch);
         assert!(batch.schema().column_with_name("i").is_some());
         assert!(batch.schema().column_with_name(DIST_COL).is_some());
-        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await;
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None, None)
+            .await;
 
         let mut scan_rowid_only = dataset.scan();
         scan_rowid_only.nearest("vec", &queries, k).unwrap();
@@ -6524,6 +7361,7 @@ mod test {
             2,
             false,
             Some((Some(1.0), None)),
+            None,
         )
         .await;
     }
@@ -6539,12 +7377,22 @@ mod test {
 
         let mut scan = dataset.scan();
         scan.nearest("vec", &queries, 2).unwrap();
+        // Probe both partitions (minimum == maximum) so the per-query top-k is
+        // merged across multiple partitions and the batch result is
+        // deterministically equivalent to repeated single-query search (which
+        // would otherwise adaptively expand nprobes).
+        scan.nprobes(2);
         scan.project(&["i"]).unwrap();
 
         let plan = scan.explain_plan(false).await.unwrap();
         assert!(
-            plan.contains("ANNSubIndex"),
-            "batch KNN should use the vector index when available, got:\n{}",
+            plan.contains("ANNIvfBatch"),
+            "IVF batch KNN should use the shared-scan batch node, got:\n{}",
+            plan
+        );
+        assert!(
+            !plan.contains("ANNSubIndex"),
+            "IVF batch KNN should not fall back to per-query ANN search, got:\n{}",
             plan
         );
         assert!(
@@ -6559,11 +7407,16 @@ mod test {
             batch[QUERY_INDEX_COL].as_primitive::<Int32Type>().values(),
             &[0, 0, 1, 1]
         );
+        // Shared-scan batch search must return the same rows/distances as
+        // issuing the queries one at a time against the index.
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, 2, true, None, Some(2))
+            .await;
 
         let batch = dataset
             .scan()
             .nearest("vec", &queries, 2)
             .unwrap()
+            .nprobes(2)
             .distance_range(Some(1.0), None)
             .project(&["i"])
             .unwrap()
@@ -6577,10 +7430,278 @@ mod test {
             2,
             true,
             Some((Some(1.0), None)),
+            Some(2),
         )
         .await;
     }
 
+    /// `refine_factor` is not yet supported by the shared-scan batch path, so
+    /// the scanner must fall back to the per-query indexed loop and still
+    /// produce correctly grouped per-query results.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_refine_falls_back() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds.make_vector_index().await.unwrap();
+        let dataset = &test_ds.dataset;
+        let (queries, _query_values) = batch_knn_two_queries();
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, 2).unwrap();
+        scan.refine(2);
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            !plan.contains("ANNIvfBatch"),
+            "refine must not use the shared-scan batch node, got:\n{}",
+            plan
+        );
+        assert!(
+            plan.contains("ANNSubIndex"),
+            "refine batch search should fall back to the per-query indexed loop, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_eq!(
+            batch[QUERY_INDEX_COL].as_primitive::<Int32Type>().values(),
+            &[0, 0, 1, 1]
+        );
+    }
+
+    /// Without pinned nprobes the shared-scan fast path is not equivalent to
+    /// single-query search (the single-query path applies an adaptive
+    /// `early_pruning` floor and late-search expansion that the batch path does
+    /// not), so the scanner must fall back to the per-query loop, which reuses
+    /// the real adaptive search and stays exact.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_adaptive_nprobes_falls_back() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds.make_vector_index().await.unwrap();
+        let dataset = &test_ds.dataset;
+        let (queries, query_values) = batch_knn_two_queries();
+        let k = 2;
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        // No nprobes() call: adaptive (minimum_nprobes=1, maximum_nprobes=None).
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            !plan.contains("ANNIvfBatch"),
+            "adaptive nprobes must not use the shared-scan batch node, got:\n{}",
+            plan
+        );
+        assert!(
+            plan.contains("ANNSubIndex"),
+            "adaptive nprobes batch search should fall back to the per-query loop, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        // The fallback runs real single-query searches, so it stays exact even
+        // with adaptive nprobes.
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, None)
+            .await;
+    }
+
+    /// IVF_HNSW is an unsupported index type for the shared-scan batch path (its
+    /// graph sub-index has no global top-k heap), so batch search must fall back
+    /// to the per-query indexed loop and still produce correct grouped results.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_hnsw_falls_back() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds.make_ivf_hnsw_index().await.unwrap();
+        let dataset = &test_ds.dataset;
+        let (queries, query_values) = batch_knn_two_queries();
+        let k = 2;
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.nprobes(2);
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            !plan.contains("ANNIvfBatch"),
+            "HNSW batch search must not use the shared-scan batch node, got:\n{}",
+            plan
+        );
+        assert!(
+            plan.contains("ANNSubIndex"),
+            "HNSW batch search should fall back to the per-query indexed loop, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, Some(2))
+            .await;
+    }
+
+    /// Regression test for cosine batch search: each query vector must be
+    /// normalized independently. The two queries below have very different
+    /// magnitudes, so normalizing the concatenated batch key by a single global
+    /// norm (the bug) would scale them unequally and diverge from per-query
+    /// single search.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_cosine_normalizes_per_query() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds
+            .make_vector_index_with_metric(MetricType::Cosine)
+            .await
+            .unwrap();
+        let dataset = &test_ds.dataset;
+
+        // q0: small-magnitude constant direction; q1: large-magnitude ramp.
+        let mut query_values = vec![0.05f32; 32];
+        query_values.extend((1..=32).map(|v| v as f32 * 3.0));
+        let queries =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(query_values.clone()), 32)
+                .unwrap();
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, 2).unwrap();
+        scan.nprobes(2);
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            plan.contains("ANNIvfBatch"),
+            "cosine IVF batch KNN should use the shared-scan batch node, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, 2, true, None, Some(2))
+            .await;
+    }
+
+    /// Batch indexed search builds a single shared prefilter for all queries;
+    /// results must match per-query single search with the same prefilter.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_with_prefilter() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds.make_vector_index().await.unwrap();
+        let dataset = &test_ds.dataset;
+        let (queries, query_values) = batch_knn_two_queries();
+        let k = 2;
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.nprobes(2);
+        scan.filter("i > 100").unwrap();
+        scan.prefilter(true);
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            plan.contains("ANNIvfBatch"),
+            "prefiltered IVF batch KNN should use the shared-scan batch node, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        // The shared prefilter must exclude i <= 100 for every query.
+        assert!(
+            batch["i"]
+                .as_primitive::<Int32Type>()
+                .values()
+                .iter()
+                .all(|i| *i > 100),
+            "shared prefilter should remove rows with i <= 100"
+        );
+
+        let query_indices = batch[QUERY_INDEX_COL].as_primitive::<Int32Type>();
+        for query_index in 0..2 {
+            let query =
+                Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec());
+            let single = dataset
+                .scan()
+                .nearest("vec", &query, k)
+                .unwrap()
+                .nprobes(2)
+                .filter("i > 100")
+                .unwrap()
+                .prefilter(true)
+                .project(&["i"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let mask = BooleanArray::from_iter(
+                query_indices
+                    .iter()
+                    .map(|v| v.map(|v| v == query_index as i32)),
+            );
+            let slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap();
+            assert_eq!(
+                slice["i"].as_primitive::<Int32Type>().values(),
+                single["i"].as_primitive::<Int32Type>().values(),
+                "prefiltered batch query {query_index} should match single-query search"
+            );
+        }
+    }
+
+    /// Batch indexed search must merge each query's top-k across multiple delta
+    /// indices, not just within a single delta.
+    #[tokio::test]
+    async fn test_batch_knn_indexed_multiple_deltas() {
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        test_ds.make_vector_index().await.unwrap();
+        // Append new data and optimize with `append` to add a second delta
+        // index (rather than merging into the existing one).
+        test_ds.append_data_with_range(400, 480).await.unwrap();
+        test_ds
+            .dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+        let dataset = &test_ds.dataset;
+        let segments = dataset.load_indices_by_name("idx").await.unwrap();
+        assert!(
+            segments.len() >= 2,
+            "expected multiple delta index segments to exercise cross-delta merge, got {}",
+            segments.len()
+        );
+
+        let (queries, query_values) = batch_knn_two_queries();
+        let k = 3;
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.nprobes(2);
+        scan.project(&["i"]).unwrap();
+
+        let plan = scan.explain_plan(false).await.unwrap();
+        assert!(
+            plan.contains("ANNIvfBatch"),
+            "multi-delta IVF batch KNN should use the shared-scan batch node, got:\n{}",
+            plan
+        );
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, Some(2))
+            .await;
+    }
+
     #[tokio::test]
     async fn test_can_project_distance() {
         let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
@@ -6878,6 +7999,57 @@ mod test {
         assert_eq!(expected_i, actual_i);
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_flat_knn_large_limit_preserves_global_order() {
+        // Regression test for https://github.com/lance-format/lance/issues/7865.
+        //
+        // An exact (flat, no vector index) KNN search with a limit larger than one
+        // output batch (BATCH_SIZE_FALLBACK = 8192 rows) used to be able to return
+        // results in the wrong global order: `execute_plan` coalesced the
+        // partitions the physical optimizer parallelizes above the top-k `SortExec`
+        // with a plain `CoalescePartitionsExec`, which does not preserve order.
+        // This only reproduces at real (> 1) parallelism, which is why the
+        // plan-shape tests elsewhere (pinned to `target_parallelism(1)`) never
+        // caught it.
+        let dim = 16u32;
+        let frag_count = 4u32;
+        let rows_per_fragment = 5_000u32;
+        let k = 12_000usize; // > BATCH_SIZE_FALLBACK, so results span multiple batches
+
+        let dataset = gen_batch()
+            .col("vec", array::rand_vec::<Float32Type>(Dimension::from(dim)))
+            .into_ram_dataset(
+                FragmentCount::from(frag_count),
+                FragmentRowCount::from(rows_per_fragment),
+            )
+            .await
+            .unwrap();
+
+        let query = Float32Array::from(vec![0.0_f32; dim as usize]);
+
+        // The bug is a scheduling race between parallel partitions, so run
+        // several iterations to reliably catch it if the ordering guarantee
+        // regresses.
+        for _ in 0..10 {
+            let mut scan = dataset.scan();
+            scan.nearest("vec", &query, k).unwrap();
+            scan.target_parallelism(8);
+
+            let batch = scan.try_into_batch().await.unwrap();
+            assert_eq!(batch.num_rows(), k);
+
+            let distances = batch[DIST_COL].as_primitive::<Float32Type>();
+            for pair in distances.values().windows(2) {
+                assert!(
+                    pair[0] <= pair[1],
+                    "flat KNN results must be globally sorted by distance, found {} before {}",
+                    pair[0],
+                    pair[1]
+                );
+            }
+        }
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_refine_factor(
@@ -7439,8 +8611,20 @@ mod test {
         #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
         data_storage_version: LanceFileVersion,
         #[values(false, true)] stable_row_ids: bool,
+        #[values(ApproxMode::Normal, ApproxMode::Fast)] approx_mode: ApproxMode,
         #[values(
             VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2),
+            VectorIndexParams::ivf_hnsw(
+                MetricType::L2,
+                IvfBuildParams::new(2),
+                HnswBuildParams::default()
+            ),
+            VectorIndexParams::with_ivf_hnsw_pq_params(
+                MetricType::L2,
+                IvfBuildParams::new(2),
+                HnswBuildParams::default(),
+                PQBuildParams::new(2, 8)
+            ),
             VectorIndexParams::with_ivf_hnsw_sq_params(
                 MetricType::L2,
                 IvfBuildParams::new(2),
@@ -7495,6 +8679,7 @@ mod test {
         scan.nearest("vector", query_key.as_ref(), 1).unwrap();
         scan.minimum_nprobes(100);
         scan.ef(100);
+        scan.approx_mode(approx_mode);
         scan.with_row_id();
 
         let batches = scan
@@ -9978,6 +11163,30 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         );
     }
 
+    /// A physical deleted-row scan cannot late-materialize: the take would
+    /// silently drop the tombstone rows (null row id), so the row-stream
+    /// read must reject the forwarded flag at plan time
+    #[tokio::test]
+    async fn test_include_deleted_rows_rejects_late_materialization() {
+        let data = gen_batch()
+            .col("i", array::step::<Int32Type>())
+            .col("payload", array::step::<Int64Type>())
+            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
+        let mut dataset = Dataset::write(data, "memory://test", None).await.unwrap();
+        dataset.delete("i = 5").await.unwrap();
+
+        let mut scan = dataset.scan();
+        scan.project(&["payload"])
+            .unwrap()
+            .filter("i > 2")
+            .unwrap()
+            .with_row_id()
+            .include_deleted_rows()
+            .materialization_style(MaterializationStyle::AllLate);
+        let err = scan.create_plan().await.unwrap_err();
+        assert!(err.to_string().contains("with_deleted_rows"), "{err}");
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_late_materialization(
@@ -10079,6 +11288,154 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_io_lt!(io_stats, read_bytes, index_scan_bytes);
     }
 
+    #[tokio::test]
+    async fn test_blob_all_binary_late_materialization() {
+        // A selective filter that projects a blob column with `blob_handling=all_binary`
+        // must late-materialize the blob (take only the matched rows) rather than eagerly
+        // reading the whole column. Blobs returned as descriptions stay eager (they are
+        // tiny), but full binary values should follow the width-based heuristic like any
+        // other wide column.
+        use lance_io::assert_io_lt;
+        use lance_table::io::commit::RenameCommitHandler;
+
+        // 8KB stays under the 64KB inline threshold, so the blob is a normal column in
+        // the data file rather than a dedicated blob file.
+        let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]);
+        let blobs = array::rand_fixedbin(ByteCount::from(8 * 1024), true).with_metadata(blob_meta);
+        let data = gen_batch()
+            .col("filterme", array::step::<UInt64Type>())
+            .col("blobs", blobs)
+            .into_reader_rows(RowCount::from(500), BatchCount::from(8));
+
+        let dataset = Dataset::write(
+            data,
+            "memory://test",
+            Some(WriteParams {
+                commit_handler: Some(Arc::new(RenameCommitHandler)),
+                data_storage_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Baseline: read the blob column as binary for the whole table.
+        let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset
+        dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .try_into_batch()
+            .await
+            .unwrap();
+        let full_scan_bytes = dataset
+            .object_store
+            .as_ref()
+            .io_stats_incremental()
+            .read_bytes;
+
+        // A filter matching a single row out of 4000 should read far less than the whole
+        // column: only the filter leaf plus the one materialized blob.
+        dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .filter("filterme = 100")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store.as_ref().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+    }
+
+    #[tokio::test]
+    async fn test_nested_blob_all_binary_late_materialization() {
+        // Same as above, but the blob is a leaf *inside* a struct and the filter is on a
+        // sibling leaf. Materialization is decided per leaf (fields_pre_order), so the
+        // nested blob must late-materialize under `all_binary` just like a top-level one.
+        use lance_io::assert_io_lt;
+        use lance_table::io::commit::RenameCommitHandler;
+
+        let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]);
+        let a_field = ArrowField::new("a", DataType::Int32, false);
+        let blob_field =
+            ArrowField::new("blob", DataType::LargeBinary, false).with_metadata(blob_meta);
+        let struct_fields: Fields = vec![a_field, blob_field].into();
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Struct(struct_fields.clone()),
+            false,
+        )]));
+
+        let rows_per_batch = 500usize;
+        let batches: Vec<RecordBatch> = (0..8)
+            .map(|b| {
+                let base = (b * rows_per_batch) as i32;
+                let a = Arc::new(Int32Array::from_iter_values(
+                    base..base + rows_per_batch as i32,
+                ));
+                // Vary the payload per row so it does not collapse under compression.
+                let blobs: Vec<Vec<u8>> = (0..rows_per_batch)
+                    .map(|r| {
+                        let seed = (base as usize + r).wrapping_mul(2654435761);
+                        (0usize..8 * 1024)
+                            .map(|i| (i.wrapping_mul(31).wrapping_add(seed) & 0xff) as u8)
+                            .collect()
+                    })
+                    .collect();
+                let blob = Arc::new(arrow_array::LargeBinaryArray::from_iter_values(
+                    blobs.iter().map(|v| v.as_slice()),
+                ));
+                let s = StructArray::new(struct_fields.clone(), vec![a, blob as ArrayRef], None);
+                RecordBatch::try_new(schema.clone(), vec![Arc::new(s)]).unwrap()
+            })
+            .collect();
+
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write(
+            reader,
+            "memory://test",
+            Some(WriteParams {
+                commit_handler: Some(Arc::new(RenameCommitHandler)),
+                data_storage_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset
+        dataset
+            .scan()
+            .project(&["s"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .try_into_batch()
+            .await
+            .unwrap();
+        let full_scan_bytes = dataset
+            .object_store
+            .as_ref()
+            .io_stats_incremental()
+            .read_bytes;
+
+        dataset
+            .scan()
+            .project(&["s"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .filter("s.a = 100")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store.as_ref().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_project_nested(
@@ -10185,9 +11542,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         LanceScan: uri..., projection=[i], row_id=true, row_addr=false, ordered=true, range=None"
         } else {
             "ProjectionExec: expr=[s@2 as s]
-  Take: columns=\"i, _rowid, (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)"
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10211,10 +11567,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                   LanceScan: uri..., projection=[i, s], row_id=true, row_addr=false, ordered=true, range=None"
         } else {
             "ProjectionExec: expr=[i@0 as i, s@1 as s, vec@3 as vec]
-  Take: columns=\"i, s, _rowid, (vec)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \
-      row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
+  LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+    LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \
+    row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10254,10 +11609,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         LanceScan: uri..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None"
         } else {
             "ProjectionExec: expr=[i@2 as i, s@0 as s, vec@3 as vec]
-  Take: columns=\"s, _rowid, (i), (vec)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \
-      range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
+  LanceRead: uri=..., projection=[i, vec], source=stream(_rowid)
+    LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \
+    range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10279,10 +11633,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         LanceScan: uri..., projection=[s, vec], row_id=true, row_addr=false, ordered=true, range=None"
         } else {
             "ProjectionExec: expr=[i@3 as i, s@0 as s, vec@1 as vec]
-  Take: columns=\"s, vec, _rowid, (i)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \
-      row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
+  LanceRead: uri=..., projection=[i], source=stream(_rowid)
+    LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \
+    row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10325,13 +11678,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance]
-  Take: columns=\"vec, _rowid, _distance, (i), (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      FilterExec: _distance@2 IS NOT NULL
-        SortExec: TopK(fetch=5), expr=...
-          KNNVectorDistance: metric=l2
-            LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
-            row_id=true, row_addr=false, full_filter=--, refine_filter=--"
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@2 IS NOT NULL
+      SortExec: TopK(fetch=5), expr=...
+        KNNVectorDistance: metric=l2
+          LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
+          row_id=true, row_addr=false, full_filter=--, refine_filter=--"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10355,14 +11707,13 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
               LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance]
-  Take: columns=\"vec, _rowid, _distance, (i), (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      GlobalLimitExec: skip=0, fetch=1
-        FilterExec: _distance@2 IS NOT NULL
-          SortExec: TopK(fetch=5), expr=...
-            KNNVectorDistance: metric=l2
-              LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
-              row_id=true, row_addr=false, full_filter=--, refine_filter=--"
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    GlobalLimitExec: skip=0, fetch=1
+      FilterExec: _distance@2 IS NOT NULL
+        SortExec: TopK(fetch=5), expr=...
+          KNNVectorDistance: metric=l2
+            LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
+            row_id=true, row_addr=false, full_filter=--, refine_filter=--"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10375,13 +11726,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         // ---------------------------------------------------------------------
         dataset.make_vector_index().await?;
         log::info!("Test case: Basic ANN");
-        let expected =
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
   Take: columns=\"_distance, _rowid, (i), (s), (vec)\"
     CoalesceBatchesExec: target_batch_size=8192
       SortExec: TopK(fetch=42), expr=...
         ANNSubIndex: name=..., k=42, deltas=1, metric=L2
-          ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1";
+          ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        } else {
+            "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
+  LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid)
+    SortExec: TopK(fetch=42), expr=...
+      ANNSubIndex: name=..., k=42, deltas=1, metric=L2
+        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| scan.nearest("vec", &q, 42),
@@ -10390,7 +11748,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: ANN with refine");
-        let expected =
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
   Take: columns=\"_rowid, vec, _distance, (i), (s)\"
     CoalesceBatchesExec: target_batch_size=8192
@@ -10401,7 +11759,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
               CoalesceBatchesExec: target_batch_size=8192
                 SortExec: TopK(fetch=40), expr=...
                   ANNSubIndex: name=..., k=40, deltas=1, metric=L2
-                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1";
+                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        } else {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=10), expr=...
+        KNNVectorDistance: metric=l2
+          LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+            SortExec: TopK(fetch=40), expr=...
+              ANNSubIndex: name=..., k=40, deltas=1, metric=L2
+                ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| Ok(scan.nearest("vec", &q, 10)?.refine(4)),
@@ -10421,13 +11790,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance]
-  Take: columns=\"vec, _rowid, _distance, (i), (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      FilterExec: _distance@... IS NOT NULL
-        SortExec: TopK(fetch=13), expr=...
-          KNNVectorDistance: metric=l2
-            LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
-            row_id=true, row_addr=false, full_filter=--, refine_filter=--"
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=13), expr=...
+        KNNVectorDistance: metric=l2
+          LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \
+          row_id=true, row_addr=false, full_filter=--, refine_filter=--"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10437,7 +11805,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: ANN with postfilter");
-        let expected = "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid]
   Take: columns=\"_distance, _rowid, i, (s), (vec)\"
     CoalesceBatchesExec: target_batch_size=8192
       FilterExec: i@2 > 10
@@ -10445,7 +11814,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
           CoalesceBatchesExec: target_batch_size=8192
             SortExec: TopK(fetch=17), expr=...
               ANNSubIndex: name=..., k=17, deltas=1, metric=L2
-                ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1";
+                ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        } else {
+            "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid]
+  LanceRead: uri=..., projection=[s, vec], source=stream(_rowid)
+    FilterExec: i@2 > 10
+      LanceRead: uri=..., projection=[i], source=stream(_rowid)
+        SortExec: TopK(fetch=17), expr=...
+          ANNSubIndex: name=..., k=17, deltas=1, metric=L2
+            ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10471,13 +11849,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
-  Take: columns=\"_distance, _rowid, (i), (s), (vec)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      SortExec: TopK(fetch=17), expr=...
-        ANNSubIndex: name=..., k=17, deltas=1, metric=L2
-          ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-          LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \
-          row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)
+  LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid)
+    SortExec: TopK(fetch=17), expr=...
+      ANNSubIndex: name=..., k=17, deltas=1, metric=L2
+        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+        LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \
+        row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)
 "
         };
         assert_plan_equals(
@@ -10494,7 +11871,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
 
         dataset.append_new_data().await?;
         log::info!("Test case: Combined KNN/ANN");
-        let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
   Take: columns=\"_rowid, vec, _distance, (i), (s)\"
     CoalesceBatchesExec: target_batch_size=8192
       FilterExec: _distance@... IS NOT NULL
@@ -10511,7 +11889,25 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                   CoalesceBatchesExec: target_batch_size=8192
                     SortExec: TopK(fetch=6), expr=...
                       ANNSubIndex: name=..., k=6, deltas=1, metric=L2
-                        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1";
+                        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        } else {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=6), expr=...
+        KNNVectorDistance: metric=l2
+          CoalescePartitionsExec
+            UnionExec
+              ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec]
+                FilterExec: _distance@... IS NOT NULL
+                  SortExec: TopK(fetch=6), expr=...
+                    KNNVectorDistance: metric=l2
+                      LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None
+              LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+                SortExec: TopK(fetch=6), expr=...
+                  ANNSubIndex: name=..., k=6, deltas=1, metric=L2
+                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| scan.nearest("vec", &q, 6),
@@ -10523,7 +11919,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
 
         // new data and with filter
         log::info!("Test case: Combined KNN/ANN with postfilter");
-        let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
   Take: columns=\"_rowid, vec, _distance, i, (s)\"
     CoalesceBatchesExec: target_batch_size=8192
       FilterExec: i@3 > 10
@@ -10543,7 +11940,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                         CoalesceBatchesExec: target_batch_size=8192
                           SortExec: TopK(fetch=15), expr=...
                             ANNSubIndex: name=..., k=15, deltas=1, metric=L2
-                              ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1";
+                              ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        } else {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    FilterExec: i@3 > 10
+      LanceRead: uri=..., projection=[i], source=stream(_rowid)
+        FilterExec: _distance@... IS NOT NULL
+          SortExec: TopK(fetch=15), expr=...
+            KNNVectorDistance: metric=l2
+              CoalescePartitionsExec
+                UnionExec
+                  ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec]
+                    FilterExec: _distance@... IS NOT NULL
+                      SortExec: TopK(fetch=15), expr=...
+                        KNNVectorDistance: metric=l2
+                          LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None
+                  LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+                    SortExec: TopK(fetch=15), expr=...
+                      ANNSubIndex: name=..., k=15, deltas=1, metric=L2
+                        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| scan.nearest("vec", &q, 15)?.filter("i > 10"),
@@ -10577,26 +11994,24 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                           LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
-  Take: columns=\"_rowid, vec, _distance, (i), (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      FilterExec: _distance@... IS NOT NULL
-        SortExec: TopK(fetch=5), expr=...
-          KNNVectorDistance: metric=l2
-            CoalescePartitionsExec
-              UnionExec
-                ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
-                  FilterExec: _distance@... IS NOT NULL
-                    SortExec: TopK(fetch=5), expr=...
-                      KNNVectorDistance: metric=l2
-                        FilterExec: i@1 > 10
-                          LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None
-                Take: columns=\"_distance, _rowid, (vec)\"
-                  CoalesceBatchesExec: target_batch_size=8192
-                    SortExec: TopK(fetch=5), expr=...
-                      ANNSubIndex: name=..., k=5, deltas=1, metric=L2
-                        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-                        LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \
-                          row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)"
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=5), expr=...
+        KNNVectorDistance: metric=l2
+          CoalescePartitionsExec
+            UnionExec
+              ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
+                FilterExec: _distance@... IS NOT NULL
+                  SortExec: TopK(fetch=5), expr=...
+                    KNNVectorDistance: metric=l2
+                      FilterExec: i@1 > 10
+                        LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None
+              LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+                SortExec: TopK(fetch=5), expr=...
+                  ANNSubIndex: name=..., k=5, deltas=1, metric=L2
+                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+                    LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \
+                      row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10619,14 +12034,22 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         dataset.make_scalar_index().await?;
 
         log::info!("Test case: ANN with scalar index");
-        let expected =
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
   Take: columns=\"_distance, _rowid, (i), (s), (vec)\"
     CoalesceBatchesExec: target_batch_size=8192
       SortExec: TopK(fetch=5), expr=...
         ANNSubIndex: name=..., k=5, deltas=1, metric=L2
           ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-          ScalarIndexQuery: query=[i > 10]@i_idx(BTree)";
+          ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        } else {
+            "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
+  LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid)
+    SortExec: TopK(fetch=5), expr=...
+      ANNSubIndex: name=..., k=5, deltas=1, metric=L2
+        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10651,13 +12074,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance]
-  Take: columns=\"_distance, _rowid, (i), (s), (vec)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      SortExec: TopK(fetch=5), expr=...
-        ANNSubIndex: name=..., k=5, deltas=1, metric=L2
-          ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-          LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \
-          range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)"
+  LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid)
+    SortExec: TopK(fetch=5), expr=...
+      ANNSubIndex: name=..., k=5, deltas=1, metric=L2
+        ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+        LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \
+        range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)"
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10675,7 +12097,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         dataset.append_new_data().await?;
 
         log::info!("Test case: Combined KNN/ANN with scalar index");
-        let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
   Take: columns=\"_rowid, vec, _distance, (i), (s)\"
     CoalesceBatchesExec: target_batch_size=8192
       FilterExec: _distance@... IS NOT NULL
@@ -10694,7 +12117,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                     SortExec: TopK(fetch=8), expr=...
                       ANNSubIndex: name=..., k=8, deltas=1, metric=L2
                         ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-                        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)";
+                        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        } else {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=8), expr=...
+        KNNVectorDistance: metric=l2
+          CoalescePartitionsExec
+            UnionExec
+              ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
+                FilterExec: _distance@... IS NOT NULL
+                  SortExec: TopK(fetch=8), expr=...
+                    KNNVectorDistance: metric=l2
+                      FilterExec: i@1 > 10
+                        LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None
+              LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+                SortExec: TopK(fetch=8), expr=...
+                  ANNSubIndex: name=..., k=8, deltas=1, metric=L2
+                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+                    ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10711,7 +12154,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         log::info!(
             "Test case: Combined KNN/ANN with updated scalar index and outdated vector index"
         );
-        let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
   Take: columns=\"_rowid, vec, _distance, (i), (s)\"
     CoalesceBatchesExec: target_batch_size=8192
       FilterExec: _distance@... IS NOT NULL
@@ -10730,7 +12174,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                     SortExec: TopK(fetch=11), expr=...
                       ANNSubIndex: name=..., k=11, deltas=1, metric=L2
                         ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
-                        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)";
+                        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        } else {
+            "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance]
+  LanceRead: uri=..., projection=[i, s], source=stream(_rowid)
+    FilterExec: _distance@... IS NOT NULL
+      SortExec: TopK(fetch=11), expr=...
+        KNNVectorDistance: metric=l2
+          CoalescePartitionsExec
+            UnionExec
+              ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
+                FilterExec: _distance@... IS NOT NULL
+                  SortExec: TopK(fetch=11), expr=...
+                    KNNVectorDistance: metric=l2
+                      FilterExec: i@1 > 10
+                        LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None
+              LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+                SortExec: TopK(fetch=11), expr=...
+                  ANNSubIndex: name=..., k=11, deltas=1, metric=L2
+                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1
+                    ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"
+        };
         dataset.make_scalar_index().await?;
         assert_plan_equals(
             &dataset.dataset,
@@ -10776,10 +12240,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                         .filter("i > 10")
                 },
                 "ProjectionExec: expr=[s@2 as s]
-  Take: columns=\"i, _rowid, (s)\"
-    CoalesceBatchesExec: target_batch_size=8192
-      LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \
-      range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)",
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \
+    range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)",
             )
             .await?;
         }
@@ -10888,10 +12351,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         // All rows are indexed
         dataset.make_fts_index().await?;
         log::info!("Test case: Full text search (match query)");
-        let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello"#;
+      MatchQuery: column=s, query=[hello]"#
+        } else {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    MatchQuery: column=s, query=[hello]"#
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10904,10 +12373,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: Full text search (phrase query)");
-        let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      PhraseQuery: column=s, query=hello world"#;
+      PhraseQuery: column=s, query=hello world"#
+        } else {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    PhraseQuery: column=s, query=hello world"#
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10921,12 +12396,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: Full text search (boost query)");
-        let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
       BoostQuery: negative_boost=1
-        MatchQuery: column=s, query=hello
-        MatchQuery: column=s, query=world"#;
+        MatchQuery: column=s, query=[hello]
+        MatchQuery: column=s, query=[world]"#
+        } else {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    BoostQuery: negative_boost=1
+      MatchQuery: column=s, query=[hello]
+      MatchQuery: column=s, query=[world]"#
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10948,7 +12431,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello
+      MatchQuery: column=s, query=[hello]
         CoalescePartitionsExec
           UnionExec
             MaterializeIndex: query=[i > 10]@i_idx(BTree)
@@ -10957,11 +12440,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                 LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"#
         } else {
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
-  Take: columns="_rowid, _score, (s)"
-    CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello
-        LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
-          ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    MatchQuery: column=s, query=[hello]
+      LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
+        ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10988,19 +12470,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
         CoalescePartitionsExec
           UnionExec
-            MatchQuery: column=s, query=hello
+            MatchQuery: column=s, query=[hello]
             FlatMatchQuery: column=s, query=hello
               LanceScan: uri=..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None"#
         } else {
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
-  Take: columns="_rowid, _score, (s)"
-    CoalesceBatchesExec: target_batch_size=8192
-      SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
-        CoalescePartitionsExec
-          UnionExec
-            MatchQuery: column=s, query=hello
-            FlatMatchQuery: column=s, query=hello
-              LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"#
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
+      CoalescePartitionsExec
+        UnionExec
+          MatchQuery: column=s, query=[hello]
+          FlatMatchQuery: column=s, query=hello
+            LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"#
         };
         dataset.append_new_data().await?;
         assert_plan_equals(
@@ -11015,10 +12496,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: Full text search with unindexed rows and fast_search");
-        let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello"#;
+      MatchQuery: column=s, query=[hello]"#
+        } else {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    MatchQuery: column=s, query=[hello]"#
+        };
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -11045,7 +12532,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
         CoalescePartitionsExec
           UnionExec
-            MatchQuery: column=s, query=hello
+            MatchQuery: column=s, query=[hello]
               CoalescePartitionsExec
                 UnionExec
                   MaterializeIndex: query=[i > 10]@i_idx(BTree)
@@ -11063,17 +12550,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                       LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"#
         } else {
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
-  Take: columns="_rowid, _score, (s)"
-    CoalesceBatchesExec: target_batch_size=8192
-      SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
-        CoalescePartitionsExec
-          UnionExec
-            MatchQuery: column=s, query=hello
-              LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
-                ScalarIndexQuery: query=[i > 10]@i_idx(BTree)
-            FlatMatchQuery: column=s, query=hello
-              LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
-                ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
+  LanceRead: uri=..., projection=[s], source=stream(_rowid)
+    SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
+      CoalescePartitionsExec
+        UnionExec
+          MatchQuery: column=s, query=[hello]
+            LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
+              ScalarIndexQuery: query=[i > 10]@i_idx(BTree)
+          FlatMatchQuery: column=s, query=hello
+            LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
+              ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -11150,11 +12636,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
                 SortExec: TopK(fetch=34), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]...
                   KNNVectorDistance: metric=l2
                     LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None
-            Take: columns=\"_distance, _rowid, (vec)\"
-              CoalesceBatchesExec: target_batch_size=8192
-                SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]...
-                  ANNSubIndex: name=idx, k=34, deltas=1, metric=L2
-                    ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1",
+            LanceRead: uri=..., projection=[vec], source=stream(_rowid)
+              SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]...
+                ANNSubIndex: name=idx, k=34, deltas=1, metric=L2
+                  ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1",
         )
         .await
         .unwrap();
@@ -12334,7 +13819,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
     }
 
     fn find_filtered_read(plan: &dyn ExecutionPlan) -> Option<&FilteredReadExec> {
-        if let Some(f) = plan.as_any().downcast_ref::<FilteredReadExec>() {
+        if let Some(f) = plan.downcast_ref::<FilteredReadExec>() {
             return Some(f);
         }
         for child in plan.children() {
@@ -12371,6 +13856,47 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_eq!(filtered.options().io_buffer_size_bytes, Some(7777));
     }
 
+    #[tokio::test]
+    async fn test_batch_readahead_bounds_decode_concurrency() {
+        let data = lance_datagen::gen_batch()
+            .col("x", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(8), BatchCount::from(1));
+        let dataset = Dataset::write(data, "memory://test_batch_readahead_concurrency", None)
+            .await
+            .unwrap();
+
+        // Default: threading mode falls back to get_num_compute_intensive_cpus().
+        let plan = dataset.scan().create_plan().await.unwrap();
+        let filtered = find_filtered_read(plan.as_ref())
+            .expect("expected a FilteredReadExec in the scan plan");
+        assert_eq!(
+            filtered.options().threading_mode,
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(get_num_compute_intensive_cpus()),
+        );
+
+        // Explicit batch_readahead(N) bounds the decode fan-out to N.
+        let mut scanner = dataset.scan();
+        scanner.batch_readahead(3);
+        let plan = scanner.create_plan().await.unwrap();
+        let filtered = find_filtered_read(plan.as_ref())
+            .expect("expected a FilteredReadExec in the scan plan");
+        assert_eq!(
+            filtered.options().threading_mode,
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(3),
+        );
+
+        let mut scanner = dataset.scan();
+        scanner.batch_readahead(0);
+        let Err(Error::InvalidInput { source, .. }) = scanner.create_plan().await else {
+            panic!("expected batch_readahead=0 to be rejected");
+        };
+        assert!(
+            source
+                .to_string()
+                .contains("batch_readahead must be greater than 0")
+        );
+    }
+
     // The env var key scopes serial_test's lock so this test only blocks others
     // that touch LANCE_DEFAULT_IO_BUFFER_SIZE — unrelated tests still run in
     // parallel.
diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs
index ce32362f324..40eac95e919 100644
--- a/rust/lance/src/dataset/schema_evolution.rs
+++ b/rust/lance/src/dataset/schema_evolution.rs
@@ -1957,6 +1957,7 @@ mod test {
                     Ok(Some(Fragment {
                         files: vec![],
                         id: 0,
+                        overlays: vec![],
                         deletion_file: None,
                         row_id_meta: None,
                         physical_rows: Some(50),
diff --git a/rust/lance/src/dataset/schema_evolution/optimize.rs b/rust/lance/src/dataset/schema_evolution/optimize.rs
index cdfdb82b87b..19bb6a6d46c 100644
--- a/rust/lance/src/dataset/schema_evolution/optimize.rs
+++ b/rust/lance/src/dataset/schema_evolution/optimize.rs
@@ -70,7 +70,7 @@ impl SqlToAllNullsOptimizer {
         match expr {
             Expr::Cast(cast) => {
                 if matches!(cast.expr.as_ref(), Expr::Literal(ScalarValue::Null, _)) {
-                    let data_type = cast.data_type.clone();
+                    let data_type = cast.field.data_type().clone();
                     AllNullsResult::AllNulls(data_type)
                 } else {
                     AllNullsResult::NotAllNulls
diff --git a/rust/lance/src/dataset/tests/dataset_aggregate.rs b/rust/lance/src/dataset/tests/dataset_aggregate.rs
index 5e55c860f5d..81aa945527d 100644
--- a/rust/lance/src/dataset/tests/dataset_aggregate.rs
+++ b/rust/lance/src/dataset/tests/dataset_aggregate.rs
@@ -22,7 +22,7 @@ use datafusion_substrait::substrait::proto::{
         reference_segment::{self, StructField},
     },
     extensions::{
-        SimpleExtensionDeclaration, SimpleExtensionUri,
+        SimpleExtensionDeclaration, SimpleExtensionUrn,
         simple_extension_declaration::{ExtensionFunction, MappingType},
     },
     function_argument::ArgType,
@@ -95,17 +95,6 @@ fn create_aggregate_rel(
             git_hash: String::new(),
             producer: "lance-test".to_string(),
         }),
-        #[allow(deprecated)]
-        extension_uris: vec![
-            SimpleExtensionUri {
-                extension_uri_anchor: 1,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
-            },
-            SimpleExtensionUri {
-                extension_uri_anchor: 2,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(),
-            },
-        ],
         extensions,
         relations: vec![PlanRel {
             rel_type: Some(datafusion_substrait::substrait::proto::plan_rel::RelType::Root(
@@ -117,7 +106,16 @@ fn create_aggregate_rel(
         }],
         advanced_extensions: None,
         expected_type_urls: vec![],
-        extension_urns: vec![],
+        extension_urns: vec![
+            SimpleExtensionUrn {
+                extension_urn_anchor: 1,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
+            },
+            SimpleExtensionUrn {
+                extension_urn_anchor: 2,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(),
+            },
+        ],
         parameter_bindings: vec![],
         type_aliases: vec![],
     };
@@ -129,9 +127,7 @@ fn create_aggregate_rel(
 fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration {
     SimpleExtensionDeclaration {
         mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-            #[allow(deprecated)]
-            extension_uri_reference: 1,
-            extension_urn_reference: 0,
+            extension_urn_reference: 1,
             function_anchor: anchor,
             name: name.to_string(),
         })),
@@ -1602,7 +1598,7 @@ async fn test_scanner_count_rows_with_fts() {
     assert_plan_node_equals(
         plan.clone(),
         "AggregateExec: mode=Single, gby=[], aggr=[count(Int32(1))]
-  MatchQuery: column=text, query=document",
+  MatchQuery: column=text, query=[document]",
     )
     .await
     .unwrap();
diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs
index 961b381e452..b12a7198e5a 100644
--- a/rust/lance/src/dataset/tests/dataset_index.rs
+++ b/rust/lance/src/dataset/tests/dataset_index.rs
@@ -863,6 +863,148 @@ async fn test_fts_on_multiple_columns() {
     assert_eq!(results.num_rows(), 1);
 }
 
+fn nested_fts_batch(
+    ids: Vec<u64>,
+    a_values: Vec<Option<&str>>,
+    b_values: Vec<Option<&str>>,
+) -> RecordBatch {
+    let a_values = Arc::new(StringArray::from(a_values)) as ArrayRef;
+    let b_values = Arc::new(StringArray::from(b_values)) as ArrayRef;
+    let struct_array = StructArray::from(vec![
+        (
+            Arc::new(Field::new("a", DataType::Utf8, true)),
+            a_values.clone(),
+        ),
+        (
+            Arc::new(Field::new("b", DataType::Utf8, true)),
+            b_values.clone(),
+        ),
+    ]);
+    let struct_type = struct_array.data_type().clone();
+    RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::UInt64, false),
+            Field::new("s", struct_type, true),
+        ])),
+        vec![
+            Arc::new(UInt64Array::from(ids)) as ArrayRef,
+            Arc::new(struct_array) as ArrayRef,
+        ],
+    )
+    .unwrap()
+}
+
+async fn nested_fts_result_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec<u64> {
+    let batch = dataset
+        .scan()
+        .full_text_search(query)
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let mut ids = batch["id"].as_primitive::<UInt64Type>().values().to_vec();
+    ids.sort_unstable();
+    ids
+}
+
+#[tokio::test]
+async fn test_fts_on_nested_fields() {
+    let batch = nested_fts_batch(
+        vec![0, 1, 2, 3],
+        vec![
+            Some("lance nested alpha"),
+            Some("plain text"),
+            None,
+            Some("phrase target here"),
+        ],
+        vec![
+            Some("metadata only"),
+            Some("database nested beta"),
+            Some("lance beta"),
+            Some("other"),
+        ],
+    );
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let test_uri = TempStrDir::default();
+    let mut dataset = Dataset::write(batches, &test_uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["s.a"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default().with_position(true),
+            true,
+        )
+        .await
+        .unwrap();
+    dataset
+        .create_index(
+            &["s.b"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let indices = dataset.load_indices().await.unwrap();
+    let indexed_fields = indices
+        .iter()
+        .map(|index| dataset.schema().field_path(index.fields[0]).unwrap())
+        .collect::<HashSet<_>>();
+    assert_eq!(
+        indexed_fields,
+        HashSet::from(["s.a".to_string(), "s.b".to_string()])
+    );
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+        MatchQuery::new("alpha".to_owned()).with_column(Some("s.a".to_owned())),
+    ));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0]);
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+        MatchQuery::new("beta".to_owned()).with_column(Some("s.b".to_owned())),
+    ));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![1, 2]);
+
+    assert_eq!(
+        nested_fts_result_ids(&dataset, FullTextSearchQuery::new("lance".to_owned())).await,
+        vec![0, 2]
+    );
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::MultiMatch(MultiMatchQuery {
+        match_queries: vec![
+            MatchQuery::new("nested".to_owned()).with_column(Some("s.a".to_owned())),
+            MatchQuery::new("nested".to_owned()).with_column(Some("s.b".to_owned())),
+        ],
+    }));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0, 1]);
+
+    let query = FullTextSearchQuery::new_query(
+        PhraseQuery::new("phrase target".to_owned())
+            .with_column(Some("s.a".to_owned()))
+            .into(),
+    );
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![3]);
+
+    let append_batch = nested_fts_batch(
+        vec![4, 5],
+        vec![Some("fresh lance append"), Some("plain append")],
+        vec![Some("other"), Some("fresh beta append")],
+    );
+    let schema = append_batch.schema();
+    let batches = RecordBatchIterator::new(vec![append_batch].into_iter().map(Ok), schema);
+    dataset.append(batches, None).await.unwrap();
+
+    assert_eq!(
+        nested_fts_result_ids(&dataset, FullTextSearchQuery::new("fresh".to_owned())).await,
+        vec![4, 5]
+    );
+}
+
 #[tokio::test]
 async fn test_fts_unindexed_data() {
     let params = InvertedIndexParams::default();
@@ -1803,6 +1945,169 @@ async fn test_fts_index_with_large_string() {
     test_fts_index::<i64, i64>(true).await;
 }
 
+#[tokio::test]
+async fn test_fts_list_index_uses_row_level_documents() {
+    let tempdir = TempStrDir::default();
+    let uri = tempdir.to_owned();
+    drop(tempdir);
+
+    let mut list_col = GenericListBuilder::<i32, _>::new(GenericStringBuilder::<i32>::new());
+    list_col.values().append_value("lance");
+    list_col.values().append_value("lance database");
+    list_col.append(true);
+    list_col.values().append_value("database");
+    list_col.append(true);
+    list_col.append(true);
+    list_col.values().append_null();
+    list_col.append(true);
+    list_col.append(false);
+
+    let docs = Arc::new(list_col.finish()) as ArrayRef;
+    let ids = Arc::new(UInt64Array::from_iter_values(0..docs.len() as u64)) as ArrayRef;
+    let batch = RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("doc", docs.data_type().clone(), true),
+            ArrowField::new("id", DataType::UInt64, false),
+        ])),
+        vec![docs, ids],
+    )
+    .unwrap();
+    let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new("lance".to_owned()).limit(Some(10)))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[0]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new("database".to_owned()).limit(Some(10)))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let mut ids = result["id"]
+        .as_primitive::<UInt64Type>()
+        .values()
+        .iter()
+        .copied()
+        .collect::<Vec<_>>();
+    ids.sort_unstable();
+    assert_eq!(ids, vec![0, 1], "{:?}", result);
+}
+
+#[tokio::test]
+async fn test_fts_list_phrase_query_can_cross_elements() {
+    assert_fts_list_phrase_query_can_cross_elements::<i32>().await;
+}
+
+#[tokio::test]
+async fn test_fts_large_list_phrase_query_can_cross_elements() {
+    assert_fts_list_phrase_query_can_cross_elements::<i64>().await;
+}
+
+async fn assert_fts_list_phrase_query_can_cross_elements<Offset: arrow::array::OffsetSizeTrait>() {
+    let tempdir = TempStrDir::default();
+    let uri = tempdir.to_owned();
+    drop(tempdir);
+
+    let mut list_col = GenericListBuilder::<Offset, _>::new(GenericStringBuilder::<Offset>::new());
+    let rows: &[&[&str]] = &[
+        &["alpha", "beta"],
+        &["want the", "apple"],
+        &["want", "apple"],
+    ];
+    for values in rows.iter().copied() {
+        for value in values {
+            list_col.values().append_value(value);
+        }
+        list_col.append(true);
+    }
+
+    let docs = Arc::new(list_col.finish()) as ArrayRef;
+    let ids = Arc::new(UInt64Array::from(vec![0u64, 1, 2])) as ArrayRef;
+    let batch = RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("doc", docs.data_type().clone(), true),
+            ArrowField::new("id", DataType::UInt64, false),
+        ])),
+        vec![docs, ids],
+    )
+    .unwrap();
+    let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    let params = InvertedIndexParams::default()
+        .with_position(true)
+        .remove_stop_words(true);
+    dataset
+        .create_index(&["doc"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("alpha beta".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[0]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("want the apple".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[1]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("want apple".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[2]);
+}
+
 #[tokio::test]
 async fn test_fts_accented_chars() {
     let ds = create_fts_dataset::<i32, i32>(false, false, InvertedIndexParams::default()).await;
@@ -3808,3 +4113,50 @@ async fn test_manifest_read_recovers_from_stale_size() {
     assert_eq!(indices.len(), 1);
     assert_eq!(indices[0].name, "id_idx");
 }
+
+/// `load_segment_params` must match the fully opened segment's params,
+/// including `custom_stop_words` — the field `InvertedIndexDetails` loses.
+#[tokio::test]
+async fn test_load_segment_params_full_fidelity() {
+    use crate::index::DatasetIndexInternalExt;
+    use lance_index::metrics::NoOpMetricsCollector;
+    use lance_index::scalar::inverted::InvertedIndex;
+
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![Field::new("text", DataType::Utf8, false)]).into(),
+        vec![Arc::new(StringArray::from(vec![
+            "the quick brown fox",
+            "lazy dogs sleep",
+        ]))],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(stream, "memory://test/segment_params", None)
+        .await
+        .unwrap();
+
+    let params = InvertedIndexParams::default().custom_stop_words(Some(vec!["quick".to_string()]));
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    let segments = crate::index::scalar::load_segments(&dataset, "text")
+        .await
+        .unwrap()
+        .expect("FTS index segments");
+    let read = crate::index::scalar::load_segment_params(&dataset, &segments[0])
+        .await
+        .unwrap();
+
+    let generic = dataset
+        .open_generic_index("text", &segments[0].uuid, &NoOpMetricsCollector)
+        .await
+        .unwrap();
+    let opened = generic
+        .as_any()
+        .downcast_ref::<InvertedIndex>()
+        .expect("inverted index");
+    assert_eq!(&read, opened.params());
+}
diff --git a/rust/lance/src/dataset/tests/dataset_io.rs b/rust/lance/src/dataset/tests/dataset_io.rs
index 1f8c7226bf2..062f0452629 100644
--- a/rust/lance/src/dataset/tests/dataset_io.rs
+++ b/rust/lance/src/dataset/tests/dataset_io.rs
@@ -12,6 +12,7 @@ use crate::dataset::WriteMode::Overwrite;
 use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::{ManifestWriteConfig, write_manifest_file};
 use crate::session::Session;
+use crate::session::caches::ManifestKey;
 use crate::{Dataset, Error, Result};
 use lance_table::format::DataStorageFormat;
 
@@ -871,6 +872,76 @@ async fn test_load_manifest_iops() {
     assert_io_eq!(io_stats, read_iops, 1);
 }
 
+#[tokio::test]
+async fn test_checkout_removed_version_not_served_from_cache() {
+    let test_uri = TempStrDir::default();
+    let session = Arc::new(Session::default());
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from_iter_values(0..10_i32))],
+    )
+    .unwrap();
+    let dataset = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        &test_uri,
+        Some(WriteParams {
+            session: Some(session.clone()),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let version = dataset.manifest().version;
+    let location = dataset.manifest_location().clone();
+    let cache = session.metadata_cache.for_dataset(&dataset.uri);
+
+    assert!(
+        cache
+            .get_with_key(&ManifestKey {
+                version,
+                e_tag: location.e_tag.as_deref(),
+            })
+            .await
+            .is_some(),
+        "manifest should be cached after the write"
+    );
+    dataset.checkout_version(version).await.unwrap();
+
+    // Remove the version from storage, as cleanup (or a manual delete) would.
+    dataset.object_store.delete(&location.path).await.unwrap();
+
+    let resolved = dataset
+        .commit_handler
+        .resolve_version_location(&dataset.base, version, &dataset.object_store.inner)
+        .await
+        .unwrap();
+    assert!(
+        resolved.size.is_none(),
+        "resolving a removed version must fall back to a size-less location, got {:?}",
+        resolved.size
+    );
+
+    cache
+        .insert_with_key(
+            &ManifestKey {
+                version,
+                e_tag: None,
+            },
+            Arc::new(dataset.manifest().clone()),
+        )
+        .await;
+    assert!(
+        dataset.checkout_version(version).await.is_err(),
+        "checkout of a version removed from storage must not be served from cache"
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn test_write_params(
@@ -1453,6 +1524,96 @@ async fn test_deep_clone(
     assert_eq!(count_files(store, &dst_root, "_deletions").await, 0);
 }
 
+// Uses an in-memory source store to force a cross-store copy. The in-memory store has
+// known platform-specific quirks on Windows (it reads back empty there; see the note in
+// tests/resource_tests.rs), so this test is gated to non-Windows. The local write side is
+// covered on Windows by `test_deep_clone` (same-store), and the cross-store streaming path
+// against real cloud stores is platform-agnostic std/tokio I/O.
+#[cfg(not(windows))]
+#[rstest]
+#[tokio::test]
+async fn test_deep_clone_cross_store(
+    #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+    data_storage_version: LanceFileVersion,
+) {
+    // Source lives in an in-memory store while the target is a local directory, so the
+    // two stores have different `store_prefix`es and `deep_clone` must stream files from
+    // the source store to the target store (the cross-account code path).
+    let session = Arc::new(Session::default());
+    let test_dir = TempStdDir::default();
+    let clone_dir = test_dir.join("clone_ds");
+    let cloned_uri = clone_dir.to_str().unwrap();
+
+    // 64 rows across 4 files exercises the multi-fragment copy path.
+    let data_reader = gen_batch()
+        .col("id", array::step::<Int32Type>())
+        .col("val", array::fill_utf8("deep".to_string()))
+        .into_reader_rows(RowCount::from(64), BatchCount::from(1));
+
+    let mut dataset = Dataset::write(
+        data_reader,
+        "memory://cross_store_src",
+        Some(WriteParams {
+            max_rows_per_file: 16,
+            max_rows_per_group: 16,
+            data_storage_version: Some(data_storage_version),
+            session: Some(session.clone()),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_ne!(dataset.object_store.store_prefix, "");
+
+    // Create a scalar index so the index files and the manifest index section are also
+    // copied across stores (the index section is read through the source store at commit).
+    dataset
+        .create_index(
+            &["id"],
+            IndexType::Scalar,
+            Some("id_idx".to_string()),
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+    // Delete some rows so a deletion file is also streamed across stores.
+    dataset.delete("id < 10").await.unwrap();
+    let cloned_dataset = dataset
+        .deep_clone(cloned_uri, dataset.version().version, None)
+        .await
+        .unwrap();
+
+    // The clone targets a local store, distinct from the in-memory source.
+    assert_ne!(
+        cloned_dataset.object_store.store_prefix,
+        dataset.object_store.store_prefix
+    );
+    assert!(cloned_dataset.manifest().base_paths.is_empty());
+
+    // Re-open the clone from a fresh session to prove the files were physically copied
+    // into the target store and the clone is fully independent of the source store.
+    let reopened = DatasetBuilder::from_uri(cloned_uri).load().await.unwrap();
+    let batches = reopened
+        .scan()
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+    assert_eq!(total_rows, 54); // 64 rows - 10 deletions
+
+    // The scalar index must have been copied and resolve against the target store, with its
+    // base reference normalized to local (no external base_paths).
+    let cloned_indices = reopened.load_indices().await.unwrap();
+    assert_eq!(cloned_indices.len(), 1);
+    assert_eq!(cloned_indices.first().unwrap().name, "id_idx");
+    assert!(cloned_indices.iter().all(|idx| idx.base_id.is_none()));
+}
+
 // Helper: count files under a dataset directory (data/_indices/_deletions)
 async fn count_files(store: &ObjectStore, root: &Path, prefix: &str) -> usize {
     use futures::StreamExt;
diff --git a/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs
new file mode 100644
index 00000000000..cfabdc13266
--- /dev/null
+++ b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs
@@ -0,0 +1,1057 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! End-to-end tests for data-overlay index masking: a scalar index masks data overlay files so that
+//! queries stay correct while overlays remain (stale index hits are dropped and new
+//! matches are added by re-evaluating overlay-covered rows on the flat path).
+
+use std::sync::Arc;
+
+use futures::TryStreamExt;
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::Int32Type;
+use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+use lance_index::IndexType;
+use lance_index::scalar::FullTextSearchQuery;
+use lance_index::scalar::ScalarIndexParams;
+use lance_index::scalar::inverted::InvertedIndexParams;
+use lance_io::utils::CachedFileSize;
+use lance_linalg::distance::MetricType;
+use lance_table::format::DataFile;
+use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+use roaring::RoaringBitmap;
+use rstest::rstest;
+
+use lance_file::writer::{FileWriter, FileWriterOptions};
+
+use crate::Dataset;
+use crate::dataset::transaction::{DataOverlayGroup, Operation};
+use crate::dataset::{WriteDestination, WriteParams};
+use crate::index::DatasetIndexExt;
+use crate::index::vector::VectorIndexParams;
+
+/// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `age` (field 1) = id * 10,
+/// six rows per file (fragments 0 and 1). In-memory store so overlay files can be written
+/// with a store-relative `data/<name>.lance` path and committed against the dataset.
+async fn create_base_dataset() -> Dataset {
+    create_base_dataset_with(false).await
+}
+
+async fn create_base_dataset_with(stable_row_ids: bool) -> Dataset {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, true),
+        ArrowField::new("age", DataType::Int32, true),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from_iter_values(0..12)),
+            Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))),
+        ],
+    )
+    .unwrap();
+    let write_params = WriteParams {
+        max_rows_per_file: 6,
+        enable_stable_row_ids: stable_row_ids,
+        ..Default::default()
+    };
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    Dataset::write(reader, "memory://", Some(write_params))
+        .await
+        .unwrap()
+}
+
+async fn build_age_index(dataset: &mut Dataset) {
+    dataset
+        .create_index(
+            &["age"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+}
+
+/// Write an overlay file covering `fields` of `fragment_id` with `coverage` and the given
+/// per-field value columns, then commit it as a `DataOverlay` transaction. `name` makes
+/// the overlay file unique.
+async fn commit_overlay(
+    dataset: Dataset,
+    name: &str,
+    fragment_id: u64,
+    fields: &[i32],
+    coverage: OverlayCoverage,
+    columns: Vec<ArrayRef>,
+) -> Dataset {
+    let read_version = dataset.version().version;
+    let overlay_schema = dataset.schema().project_by_ids(fields, true);
+
+    let filename = format!("{name}.lance");
+    // Use dataset.base so the path is absolute for file:// stores.
+    // to_local_path() prepends '/' to the object_store path, so a bare
+    // "data/foo.lance" would resolve to /data/foo.lance (root fs). With
+    // base we get e.g. tmp/lance-bench/data/foo.lance → /tmp/lance-bench/data/foo.lance.
+    // For memory:// stores base is empty so the result is the same as before.
+    let path = dataset.base.clone().join("data").join(filename.as_str());
+    let obj_writer = dataset.object_store.create(&path).await.unwrap();
+    let mut writer =
+        FileWriter::try_new(obj_writer, overlay_schema, FileWriterOptions::default()).unwrap();
+    let (major, minor) = writer.version().to_numbers();
+    for (i, array) in columns.into_iter().enumerate() {
+        writer.write_column(i, array).await.unwrap();
+    }
+    let summary = writer.finish().await.unwrap();
+
+    let mut data_file = DataFile::new_unstarted(filename, major, minor);
+    data_file.fields = writer
+        .field_id_to_column_indices()
+        .iter()
+        .map(|(field_id, _)| *field_id as i32)
+        .collect::<Vec<_>>()
+        .into();
+    data_file.column_indices = writer
+        .field_id_to_column_indices()
+        .iter()
+        .map(|(_, column_index)| *column_index as i32)
+        .collect::<Vec<_>>()
+        .into();
+    data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+    let overlay = DataOverlayFile {
+        data_file,
+        coverage,
+        committed_version: 0,
+    };
+    Dataset::commit(
+        WriteDestination::Dataset(Arc::new(dataset)),
+        Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![overlay],
+            }],
+        },
+        Some(read_version),
+        None,
+        None,
+        Arc::new(Default::default()),
+        false,
+    )
+    .await
+    .unwrap()
+}
+
+/// Sorted `id` values returned by a filtered scan.
+async fn ids_matching(dataset: &Dataset, filter: &str) -> Vec<i32> {
+    ids_matching_opts(dataset, filter, false).await
+}
+
+/// Like [`ids_matching`] but lets a test enable `fast_search()`, which skips unindexed
+/// fragments. Overlay masking on indexed fragments must still apply regardless.
+async fn ids_matching_opts(dataset: &Dataset, filter: &str, fast_search: bool) -> Vec<i32> {
+    let mut scanner = dataset.scan();
+    scanner.filter(filter).unwrap().project(&["id"]).unwrap();
+    if fast_search {
+        scanner.fast_search();
+    }
+    let batch = scanner.try_into_batch().await.unwrap();
+    let mut ids = ids_from_batches(std::slice::from_ref(&batch));
+    ids.sort_unstable();
+    ids
+}
+
+/// Concatenate the `id` (Int32) column from each batch, in batch order.
+fn ids_from_batches(batches: &[RecordBatch]) -> Vec<i32> {
+    batches
+        .iter()
+        .flat_map(|b| {
+            b.column_by_name("id")
+                .unwrap()
+                .as_primitive::<Int32Type>()
+                .values()
+                .to_vec()
+        })
+        .collect()
+}
+
+fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+    Arc::new(Int32Array::from_iter(values))
+}
+
+fn fsl(rows: Vec<Vec<f32>>, dim: i32) -> ArrayRef {
+    let flat: Vec<f32> = rows.into_iter().flatten().collect();
+    let item = Arc::new(ArrowField::new("item", DataType::Float32, true));
+    Arc::new(
+        arrow_array::FixedSizeListArray::try_new(
+            item,
+            dim,
+            Arc::new(arrow_array::Float32Array::from(flat)),
+            None,
+        )
+        .unwrap(),
+    )
+}
+
+/// A newer overlay on the indexed field drops stale index hits (the old value no longer
+/// matches) and surfaces new matches (the new value is found even though the index never
+/// saw it). Mirrors the spec's Bob 25 -> 26 worked example.
+///
+/// Parametrized over `stable_row_ids` to cover the address-based stale-Take path under both
+/// row-id schemes.
+#[rstest]
+#[tokio::test]
+async fn test_overlay_stale_drop_and_new_match(#[values(false, true)] stable_row_ids: bool) {
+    let mut dataset = create_base_dataset_with(stable_row_ids).await;
+    build_age_index(&mut dataset).await;
+
+    // Fragment 0, offset 1 is id=1, age=10. The overlay (committed after the index)
+    // changes its age to 999.
+    let dataset = commit_overlay(
+        dataset,
+        "age_overlay",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(999)])],
+    )
+    .await;
+
+    // Stale-drop: the index still holds age=10 for id=1, but its current value is 999,
+    // so it must not be returned.
+    assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::<i32>::new());
+    // New-match: the index never saw age=999, but re-evaluation finds it.
+    assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]);
+    // An untouched indexed value is unaffected.
+    assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]);
+}
+
+/// Row-level BTree precision: when one row in a covered fragment is stale, only that row is
+/// blocked from the index result and re-evaluated on the stale-Take path. Non-stale rows in
+/// the same fragment (including one that matches the predicate) remain on the indexed path.
+///
+/// Setup: fragment 0 has id=5 → age=50 (not stale). Overlay id=1 → age=50 (stale).
+/// After the overlay two rows in fragment 0 have age=50. The row-level optimization must
+/// return both: id=5 from the index and id=1 from the stale-Take path.
+///
+/// Parametrized over `stable_row_ids`: with stable row ids enabled the stale-Take path must
+/// identify rows by physical address, not `_rowid`, or it would take the wrong rows.
+#[rstest]
+#[tokio::test]
+async fn test_btree_overlay_row_level_precision(#[values(false, true)] stable_row_ids: bool) {
+    let mut dataset = create_base_dataset_with(stable_row_ids).await;
+    build_age_index(&mut dataset).await;
+
+    // Fragment 0: ids 0-5, ages 0,10,20,30,40,50. Overlay offset 1 (id=1): age 10→50.
+    // After this both id=1 and id=5 have age=50, in the same fragment.
+    let dataset = commit_overlay(
+        dataset,
+        "age_row_level",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(50)])],
+    )
+    .await;
+
+    // Stale drop: id=1's old age=10 entry must not appear.
+    assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::<i32>::new());
+
+    // id=5 via index + id=1 via stale-Take path — both in fragment 0.
+    assert_eq!(ids_matching(&dataset, "age = 50").await, vec![1, 5]);
+
+    // Non-stale rows in the same fragment still return correctly.
+    assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]);
+    assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]);
+}
+
+/// `fast_search` skips *unindexed fragments*, but overlay masking on indexed fragments must
+/// still apply: the drop-stale block and the stale-Take re-eval both run regardless of
+/// `fast_search` on the scalar path. A regression that gated overlay masking behind
+/// `!fast_search` would leak id=1's stale age=10 hit here.
+#[tokio::test]
+async fn test_btree_overlay_masked_under_fast_search() {
+    let mut dataset = create_base_dataset().await;
+    build_age_index(&mut dataset).await;
+
+    // Fragment 0, offset 1 is id=1, age=10. Overlay (committed after the index) → age=999.
+    let dataset = commit_overlay(
+        dataset,
+        "age_fast_search",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(999)])],
+    )
+    .await;
+
+    // Stale hit dropped even under fast_search — the block is not gated by fast_search.
+    assert_eq!(
+        ids_matching_opts(&dataset, "age = 10", true).await,
+        Vec::<i32>::new()
+    );
+    // The scalar re-eval path is likewise not gated, so the new value is still surfaced.
+    assert_eq!(
+        ids_matching_opts(&dataset, "age = 999", true).await,
+        vec![1]
+    );
+    // An untouched indexed value on the same fragment is unaffected.
+    assert_eq!(ids_matching_opts(&dataset, "age = 20", true).await, vec![2]);
+}
+
+/// An overlay touching only a non-indexed field excludes nothing from the index on `age`.
+#[tokio::test]
+async fn test_overlay_on_unrelated_field_excludes_nothing() {
+    let mut dataset = create_base_dataset().await;
+    build_age_index(&mut dataset).await;
+
+    // Overlay field 0 (`id`), not the indexed `age`. The age index stays fully trusted.
+    let dataset = commit_overlay(
+        dataset,
+        "id_overlay",
+        0,
+        &[0],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(777)])],
+    )
+    .await;
+
+    // The age index is still trusted: age=10 finds the offset-1 row, whose id now reads
+    // through the overlay as 777. The fragment was not routed to the flat path on account
+    // of an overlay that touches no indexed field.
+    assert_eq!(ids_matching(&dataset, "age = 10").await, vec![777]);
+    // An untouched row is unaffected.
+    assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]);
+    // The overlaid id is the new value on read, and the old one is gone.
+    assert_eq!(ids_matching(&dataset, "id = 777").await, vec![777]);
+    assert_eq!(ids_matching(&dataset, "id = 1").await, Vec::<i32>::new());
+}
+
+/// An overlay whose `committed_version <= index.dataset_version` is already incorporated by
+/// the index (the index was built reading merged values) and is not excluded.
+#[tokio::test]
+async fn test_overlay_older_than_index_not_excluded() {
+    let dataset = create_base_dataset().await;
+
+    // Commit the overlay first (age of id=1 becomes 999), then build the index on top.
+    let mut dataset = commit_overlay(
+        dataset,
+        "age_overlay_old",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(999)])],
+    )
+    .await;
+    build_age_index(&mut dataset).await;
+
+    // The index incorporates the overlay, so it returns the merged value directly.
+    assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]);
+    assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::<i32>::new());
+}
+
+/// A covered offset whose overlay value is NULL overrides the cell to NULL, so the stale
+/// index hit for its old value is dropped.
+#[tokio::test]
+async fn test_overlay_null_override() {
+    let mut dataset = create_base_dataset().await;
+    build_age_index(&mut dataset).await;
+
+    // id=1 (age=10) is overridden to NULL.
+    let dataset = commit_overlay(
+        dataset,
+        "age_overlay_null",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([None])],
+    )
+    .await;
+
+    assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::<i32>::new());
+    assert_eq!(ids_matching(&dataset, "age IS NULL").await, vec![1]);
+}
+
+/// Overlays on a non-first fragment are masked correctly, and a query spanning both
+/// fragments returns the right rows.
+///
+/// Parametrized over `stable_row_ids`, and crucially overlays fragment 1 (ids 6..12), where a
+/// physical address diverges from the stable row id — so this exercises the address-vs-row-id
+/// distinction that a fragment-0 overlay cannot.
+#[rstest]
+#[tokio::test]
+async fn test_overlay_multi_fragment(#[values(false, true)] stable_row_ids: bool) {
+    let mut dataset = create_base_dataset_with(stable_row_ids).await;
+    build_age_index(&mut dataset).await;
+
+    // Fragment 1 holds ids 6..12 (ages 60..110). Offset 2 within fragment 1 is id=8,
+    // age=80; change it to 60 (a value that also legitimately exists at id=6).
+    let dataset = commit_overlay(
+        dataset,
+        "age_overlay_frag1",
+        1,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([2])),
+        vec![i32_array([Some(60)])],
+    )
+    .await;
+
+    // id=8 no longer has age=80 (stale-drop on fragment 1).
+    assert_eq!(ids_matching(&dataset, "age = 80").await, Vec::<i32>::new());
+    // Both id=6 (base) and id=8 (overlay) now have age=60 (new-match added to base hit).
+    assert_eq!(ids_matching(&dataset, "age = 60").await, vec![6, 8]);
+    // A value in the untouched fragment 0 is still served correctly.
+    assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]);
+}
+
+const VEC_DIM: i32 = 8;
+
+fn vec_query() -> Vec<f32> {
+    vec![1.0_f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+}
+
+/// 64-row two-fragment vector dataset with a single-partition IVF_FLAT index, then an overlay
+/// on fragment 1 that moves id=35 (offset 3) onto `far` (away from the query) and id=40
+/// (offset 8) onto the query. Built before the overlay, the index still believes id=35 is the
+/// query and has never seen id=40 near it. Every other base vector is orthogonal to the query.
+///
+/// Overlaying fragment 1 (ids 32..64) is deliberate: a physical address diverges from the
+/// stable row id there, so both the ANN prefilter block and the flat re-score take must operate
+/// in the row-id domain when `stable_row_ids` is enabled.
+async fn create_vector_overlay_dataset(stable_row_ids: bool) -> Dataset {
+    let query = vec_query();
+    let far = vec![0.0_f32, 100.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0];
+
+    let mut vectors: Vec<Vec<f32>> = Vec::with_capacity(64);
+    for i in 0..64 {
+        if i == 35 {
+            vectors.push(query.clone());
+        } else {
+            let mut v = vec![0.0_f32; VEC_DIM as usize];
+            v[1] = (i + 2) as f32; // orthogonal to the query, distinct, far
+            vectors.push(v);
+        }
+    }
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, true),
+        ArrowField::new(
+            "vec",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                VEC_DIM,
+            ),
+            true,
+        ),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from_iter_values(0..64)),
+            fsl(vectors, VEC_DIM),
+        ],
+    )
+    .unwrap();
+    let write_params = WriteParams {
+        max_rows_per_file: 32,
+        enable_stable_row_ids: stable_row_ids,
+        ..Default::default()
+    };
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(reader, "memory://", Some(write_params))
+        .await
+        .unwrap();
+
+    // Single-partition IVF_FLAT: the ANN searches every indexed row with exact distances.
+    let params = VectorIndexParams::ivf_flat(1, MetricType::L2);
+    dataset
+        .create_index(&["vec"], IndexType::Vector, None, &params, true)
+        .await
+        .unwrap();
+
+    commit_overlay(
+        dataset,
+        "vec_overlay",
+        1,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([3, 8])),
+        vec![fsl(vec![far, query], VEC_DIM)],
+    )
+    .await
+}
+
+/// Run a top-`k` ANN search for the standard query vector and return the returned `id`s,
+/// optionally with `fast_search()` enabled.
+async fn vector_query_ids(dataset: &Dataset, k: usize, fast_search: bool) -> Vec<i32> {
+    let mut scanner = dataset.scan();
+    scanner
+        .nearest("vec", &arrow_array::Float32Array::from(vec_query()), k)
+        .unwrap()
+        .minimum_nprobes(1)
+        .project(&["id"])
+        .unwrap();
+    if fast_search {
+        scanner.fast_search();
+    }
+    let results = scanner
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    ids_from_batches(&results)
+}
+
+/// A vector index masks overlays: a row whose vector was moved (by a newer overlay) away
+/// from the query is dropped from results, and a row moved *onto* the query is found by
+/// re-scoring its current vector on the flat path — even though the index never saw it.
+///
+/// Parametrized over `stable_row_ids` to cover the row-id domain for both block and re-score.
+#[rstest]
+#[tokio::test]
+async fn test_vector_index_rescore_on_overlay(#[values(false, true)] stable_row_ids: bool) {
+    let dataset = create_vector_overlay_dataset(stable_row_ids).await;
+    let ids = vector_query_ids(&dataset, 3, false).await;
+
+    // id=40 was moved onto the query and is found by re-scoring (new-match recall).
+    assert!(
+        ids.contains(&40),
+        "expected id=40 (re-scored to query) in {ids:?}"
+    );
+    // id=35's stale index entry (the query) must not resurface: its current vector is far.
+    assert!(
+        !ids.contains(&35),
+        "stale vector for id=35 should be dropped, got {ids:?}"
+    );
+}
+
+/// The ANN prefilter block that drops stale overlay rows runs regardless of `fast_search`;
+/// only the flat re-score is gated by it. So under `fast_search` id=35's stale hit must still
+/// be dropped, while id=40 (moved onto the query) is intentionally not re-scored — the same
+/// recall tradeoff `fast_search` already makes for unindexed data. A regression that moved the
+/// `overlay_block` computation inside the `!fast_search` guard would leak id=35's stale vector.
+#[tokio::test]
+async fn test_vector_overlay_stale_dropped_under_fast_search() {
+    let dataset = create_vector_overlay_dataset(false).await;
+    let ids = vector_query_ids(&dataset, 3, true).await;
+
+    // Correctness: the stale index hit is dropped even though the re-score is skipped.
+    assert!(
+        !ids.contains(&35),
+        "stale vector for id=35 must be dropped under fast_search, got {ids:?}"
+    );
+    // Recall tradeoff: fast_search skips the flat re-score, so the moved-on match is not surfaced.
+    assert!(
+        !ids.contains(&40),
+        "fast_search skips re-score, so id=40 should be absent, got {ids:?}"
+    );
+}
+
+/// A compound boolean predicate (age AND id) exercises the ScalarIndexExpr tree-walk in
+/// `overlay_stale_index_rows`. An overlay on `age` marks fragment 0 stale from the `age`
+/// index's perspective, so the compound query must re-evaluate fragment 0 on the flat path.
+#[tokio::test]
+async fn test_overlay_stale_with_compound_index_expression() {
+    let mut dataset = create_base_dataset().await;
+    // Build BTree indexes on both columns so a compound filter can use both.
+    build_age_index(&mut dataset).await;
+    dataset
+        .create_index(
+            &["id"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    // Fragment 0 covers id=0..5, age=0..50. Overlay changes id=1's age from 10 to 999.
+    let dataset = commit_overlay(
+        dataset,
+        "age_compound",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(999)])],
+    )
+    .await;
+
+    // Compound query: both the `age` and `id` index are involved. The overlay on `age`
+    // makes fragment 0 stale for the `age` index; it falls to the flat path, which uses
+    // the merged (overlay) value. Result: the stale age=10 hit is gone, age=999 appears.
+    assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::<i32>::new());
+    assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]);
+    // A pure `id` query on an unaffected fragment still works correctly.
+    assert_eq!(ids_matching(&dataset, "id = 2").await, vec![2]);
+}
+
+/// Text dataset: two fragments, 6 rows each. Schema: id (Int32), text (Utf8).
+/// Texts are unique tokens so each row can be identified by its term.
+async fn create_text_dataset() -> Dataset {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, true),
+        ArrowField::new("text", DataType::Utf8, true),
+    ]));
+    let texts: Vec<&str> = vec![
+        "apple pie",
+        "apple banana", // row 1, fragment 0 — will be overlaid in tests
+        "cherry cake",
+        "banana split",
+        "orange juice",
+        "grape vine",
+        "mango sorbet", // fragment 1 starts here
+        "pear tart",
+        "lemon curd",
+        "peach cobbler",
+        "plum pudding",
+        "fig newton",
+    ];
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from_iter_values(0..12)),
+            Arc::new(StringArray::from(texts)),
+        ],
+    )
+    .unwrap();
+    let write_params = WriteParams {
+        max_rows_per_file: 6,
+        ..Default::default()
+    };
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    Dataset::write(reader, "memory://", Some(write_params))
+        .await
+        .unwrap()
+}
+
+async fn build_text_fts_index(dataset: &mut Dataset) {
+    dataset
+        .create_index(
+            &["text"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+}
+
+/// FTS index with token positions stored, required for phrase queries.
+async fn build_text_fts_index_with_positions(dataset: &mut Dataset) {
+    dataset
+        .create_index(
+            &["text"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default().with_position(true),
+            true,
+        )
+        .await
+        .unwrap();
+}
+
+/// Collect sorted IDs of rows returned by an FTS query on `text`.
+async fn fts_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec<i32> {
+    let results = dataset
+        .scan()
+        .full_text_search(query)
+        .unwrap()
+        .project(&["id"])
+        .unwrap()
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    let mut ids = ids_from_batches(&results);
+    ids.sort_unstable();
+    ids
+}
+
+async fn fts_ids_matching(dataset: &Dataset, term: &str) -> Vec<i32> {
+    fts_ids(dataset, FullTextSearchQuery::new(term.to_owned())).await
+}
+
+async fn fts_phrase_ids_matching(dataset: &Dataset, phrase: &str) -> Vec<i32> {
+    use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery};
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::Phrase(
+        PhraseQuery::new(phrase.to_owned()).with_column(Some("text".to_owned())),
+    ));
+    fts_ids(dataset, query).await
+}
+
+/// An overlay committed after the FTS index is built replaces a row's text. Searching for
+/// the old term must not return the stale row; searching for the new term must find it.
+#[tokio::test]
+async fn test_fts_overlay_stale_drop_and_new_match() {
+    let mut dataset = create_text_dataset().await;
+    build_text_fts_index(&mut dataset).await;
+
+    // fragment 0, row offset 1 (id=1): "apple banana" → "cherry mango"
+    // field ID 1 is the `text` column.
+    let dataset = commit_overlay(
+        dataset,
+        "text_overlay",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))],
+    )
+    .await;
+
+    // "apple" now matches only id=0 ("apple pie"); id=1's stale index entry must be dropped.
+    assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0]);
+
+    // "banana" matched id=1 and id=3 before; after overlay id=1's stale entry must be gone.
+    assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3]);
+
+    // "cherry" now matches id=1 (via flat path on stale fragment) and id=2 ("cherry cake").
+    let cherry_ids = fts_ids_matching(&dataset, "cherry").await;
+    assert!(
+        cherry_ids.contains(&1),
+        "id=1 overlay→cherry mango should be found: {cherry_ids:?}"
+    );
+    assert!(
+        cherry_ids.contains(&2),
+        "id=2 cherry cake should still be found: {cherry_ids:?}"
+    );
+
+    // "mango" now matches id=1 (overlay) and id=6 ("mango sorbet" in fragment 1).
+    let mango_ids = fts_ids_matching(&dataset, "mango").await;
+    assert!(
+        mango_ids.contains(&1),
+        "id=1 overlay→cherry mango should be found: {mango_ids:?}"
+    );
+    assert!(
+        mango_ids.contains(&6),
+        "id=6 mango sorbet should still be found: {mango_ids:?}"
+    );
+}
+
+/// A phrase query must not return a stale hit for an overlaid FTS-indexed row. Phrase queries
+/// have no flat re-evaluation path, so the fragment is excluded from the indexed phrase search
+/// (like an unindexed fragment) rather than re-scored — the point of this test is that the
+/// pre-overlay phrase hit is dropped, not that the new value is found.
+#[tokio::test]
+async fn test_fts_phrase_overlay_stale_drop() {
+    let mut dataset = create_text_dataset().await;
+    build_text_fts_index_with_positions(&mut dataset).await;
+
+    // Before any overlay the phrase "apple banana" matches only id=1.
+    assert_eq!(
+        fts_phrase_ids_matching(&dataset, "apple banana").await,
+        vec![1]
+    );
+
+    // Overlay id=1's text (field 1) so the phrase no longer applies to its current value.
+    let dataset = commit_overlay(
+        dataset,
+        "phrase_overlay",
+        0,
+        &[1],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))],
+    )
+    .await;
+
+    // The stale inverted-index positions for "apple banana" on id=1 must not be returned.
+    assert_eq!(
+        fts_phrase_ids_matching(&dataset, "apple banana").await,
+        Vec::<i32>::new()
+    );
+}
+
+/// An overlay on a non-FTS field must not exclude the fragment from phrase search.
+#[tokio::test]
+async fn test_fts_phrase_overlay_unrelated_field_not_excluded() {
+    let mut dataset = create_text_dataset().await;
+    build_text_fts_index_with_positions(&mut dataset).await;
+
+    // Overlay field 0 (`id`), not the FTS-indexed `text` column: phrase coverage is untouched.
+    let dataset = commit_overlay(
+        dataset,
+        "id_overlay",
+        0,
+        &[0],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(777)])],
+    )
+    .await;
+
+    assert_eq!(
+        fts_phrase_ids_matching(&dataset, "apple banana").await,
+        vec![777]
+    );
+}
+
+/// An overlay on a field the FTS index does NOT cover must not exclude anything.
+#[tokio::test]
+async fn test_fts_overlay_unrelated_field_not_excluded() {
+    let mut dataset = create_text_dataset().await;
+    build_text_fts_index(&mut dataset).await;
+
+    // Overlay field 0 (id) — not covered by the FTS index on `text`.
+    let dataset = commit_overlay(
+        dataset,
+        "id_overlay_for_fts",
+        0,
+        &[0],
+        OverlayCoverage::dense(RoaringBitmap::from_iter([1])),
+        vec![i32_array([Some(999)])],
+    )
+    .await;
+
+    // FTS coverage must be unchanged — both rows containing "apple" are still returned.
+    // The `id` overlay changes row offset 1's id from 1 to 999, so the projected id column
+    // reflects the overlay even though the FTS index correctly returned that row.
+    assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0, 999]);
+    assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3, 999]);
+}
+
+/// Benchmark: measure query latency for BTree, FTS, and vector ANN with 0/4/16 overlay layers.
+///
+/// Run with: cargo test -p lance --lib --release -- overlay_index_masking::bench --ignored --nocapture
+#[tokio::test]
+#[ignore = "benchmark"]
+#[allow(clippy::print_stdout)]
+async fn bench_index_query_overlay_overhead() {
+    use std::time::Instant;
+
+    use arrow_array::Float32Array;
+
+    const DIM: i32 = 32;
+    const ROWS: i32 = 1_000_000;
+    const ROWS_PER_FRAG: i32 = 100_000; // 10 fragments
+    const ITERS: u32 = 10; // large scans — 10 is enough for stable averages
+
+    // Fixed disk path so timings are comparable across runs. Deleted and recreated fresh.
+    let uri = "/tmp/lance-bench-overlay-oss1325";
+    if std::path::Path::new(uri).exists() {
+        std::fs::remove_dir_all(uri).unwrap();
+    }
+
+    // --- Build 1M-row dataset on local disk --------------------------------
+    // Schema: id(0), age(1), vec(2) — 3 top-level fields.
+    // Lance field IDs (depth-first): id=0, age=1, vec=2, vec.item=3.
+
+    println!("Building {ROWS}-row dataset at {uri} (this takes ~30 s)...");
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("age", DataType::Int32, false),
+        ArrowField::new(
+            "vec",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                DIM,
+            ),
+            false,
+        ),
+    ]));
+
+    let row_ids: Vec<i32> = (0..ROWS).collect();
+    let ages: Vec<i32> = row_ids.iter().map(|&i| i * 10).collect();
+    // Build the 128 MB flat float array directly (avoids 1M per-row Vec allocations).
+    let flat_vecs: Vec<f32> = (0..(ROWS as usize * DIM as usize))
+        .map(|j| (j / DIM as usize) as f32 % 1000.0)
+        .collect();
+    let vec_col = Arc::new(
+        arrow_array::FixedSizeListArray::try_new(
+            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+            DIM,
+            Arc::new(Float32Array::from(flat_vecs)),
+            None,
+        )
+        .unwrap(),
+    );
+
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(row_ids)),
+            Arc::new(Int32Array::from(ages)),
+            vec_col,
+        ],
+    )
+    .unwrap();
+
+    let write_params = WriteParams {
+        max_rows_per_file: ROWS_PER_FRAG as usize,
+        ..Default::default()
+    };
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(reader, uri, Some(write_params))
+        .await
+        .unwrap();
+
+    println!("Building BTree index on age...");
+    dataset
+        .create_index(
+            &["age"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    println!("Building IVF_FLAT(1 partition) index on vec...");
+    dataset
+        .create_index(
+            &["vec"],
+            IndexType::Vector,
+            None,
+            &VectorIndexParams::ivf_flat(1, MetricType::L2),
+            true,
+        )
+        .await
+        .unwrap();
+
+    println!("Indexes built.\n");
+
+    // --- Timing helper ---------------------------------------------------
+
+    async fn timeit<F, Fut>(iters: u32, mut f: F) -> f64
+    where
+        F: FnMut() -> Fut,
+        Fut: std::future::Future<Output = ()>,
+    {
+        f().await; // warmup
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            f().await;
+        }
+        t0.elapsed().as_secs_f64() * 1000.0 / iters as f64
+    }
+
+    // === Scenario A: BTree query overhead ================================
+    //
+    // Overlay on `age` (field 1), covering only offset 0 of fragment 0.
+    // Fragment granularity: the entire fragment 0 (100k rows) falls to flat-scan.
+    //
+    // btree_cold: `age = 420` → id=42 → in fragment 0 (rows 0..99999).
+    //   With overlays: 100k-row flat scan + per-overlay merge instead of index lookup.
+    //   Without overlays: O(log n) BTree lookup.
+    //
+    // btree_warm: `age = 1000420` → id=100042 → in fragment 1 (rows 100000..199999).
+    //   Always served by the BTree index regardless of overlay count on fragment 0.
+    //   This isolates the index-lookup baseline.
+    println!("=== Scenario A: BTree (overlay on `age`, fragment 0 becomes stale) ===");
+    println!(
+        "{:>10}  {:>14}  {:>14}",
+        "overlays", "cold_frag0_ms", "warm_frag1_ms"
+    );
+
+    let mut committed_a = 0u32;
+    for num_overlays in [0u32, 1, 4, 16] {
+        // Commit only the delta since the last iteration.
+        for layer in committed_a..num_overlays {
+            dataset = commit_overlay(
+                dataset,
+                &format!("age_ol{layer}"),
+                0,    // fragment 0
+                &[1], // field 1 = age
+                OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                vec![i32_array([Some(999)])],
+            )
+            .await;
+        }
+        committed_a = num_overlays;
+
+        let ds = Arc::new(dataset.clone());
+
+        // Cold path: stale fragment falls to flat scan when overlays > 0.
+        let ds2 = ds.clone();
+        let cold_ms = timeit(ITERS, || {
+            let ds = ds2.clone();
+            async move {
+                ds.scan()
+                    .filter("age = 420")
+                    .unwrap()
+                    .project(&["age"])
+                    .unwrap()
+                    .try_into_batch()
+                    .await
+                    .unwrap();
+            }
+        })
+        .await;
+
+        // Warm path: fragment 1 never stale, always index-served.
+        let ds2 = ds.clone();
+        let warm_ms = timeit(ITERS, || {
+            let ds = ds2.clone();
+            async move {
+                ds.scan()
+                    .filter("age = 1000420")
+                    .unwrap()
+                    .project(&["age"])
+                    .unwrap()
+                    .try_into_batch()
+                    .await
+                    .unwrap();
+            }
+        })
+        .await;
+
+        println!("{num_overlays:>10}  {cold_ms:>14.1}  {warm_ms:>14.1}");
+    }
+
+    // === Scenario B: Vector ANN overhead =================================
+    //
+    // Overlay on `vec` (field 2), covering only offset 0 of fragment 0.
+    // The field-aware check means the 16 age overlays from Scenario A do NOT affect
+    // the vector index (they touch field 1, not field 2). Only a vec overlay (field 2)
+    // marks fragment 0 stale for the vector index.
+    //
+    // With a vec overlay: 100k rows of fragment 0 are excluded from ANN prefilter
+    // bitmaps and re-scored brute-force (O(100k × DIM) distance computations).
+    println!("\n=== Scenario B: Vector ANN (overlay on `vec`, 100k rows brute-forced) ===");
+    println!("{:>12}  {:>10}", "vec_overlays", "ann_ms");
+
+    let query_vec = Float32Array::from(vec![0.5f32; DIM as usize]);
+
+    for num_vec_overlays in [0u32, 1] {
+        if num_vec_overlays == 1 {
+            dataset = commit_overlay(
+                dataset,
+                "vec_ol0",
+                0,    // fragment 0
+                &[2], // field 2 = vec (FixedSizeList top-level field)
+                OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                vec![fsl(vec![vec![0.0f32; DIM as usize]], DIM)],
+            )
+            .await;
+        }
+
+        let ds = Arc::new(dataset.clone());
+        let ds2 = ds.clone();
+        let qv = query_vec.clone();
+        let ann_ms = timeit(ITERS, || {
+            let ds = ds2.clone();
+            let q = qv.clone();
+            async move {
+                ds.scan()
+                    .nearest("vec", &q, 10)
+                    .unwrap()
+                    .minimum_nprobes(1)
+                    .project(&["id"])
+                    .unwrap()
+                    .try_into_batch()
+                    .await
+                    .unwrap();
+            }
+        })
+        .await;
+
+        println!("{num_vec_overlays:>12}  {ann_ms:>10.1}");
+    }
+}
diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs
index 3e2a4caa3b3..74790d301c1 100644
--- a/rust/lance/src/dataset/tests/dataset_transactions.rs
+++ b/rust/lance/src/dataset/tests/dataset_transactions.rs
@@ -272,6 +272,39 @@ pub(super) fn assert_results<T: Array + PartialEq + 'static>(
     )
 }
 
+fn gen_rows() -> impl arrow_array::RecordBatchReader + Send + 'static {
+    lance_datagen::gen_batch()
+        .col("key", array::step::<Int32Type>())
+        .into_reader_rows(RowCount::from(10), BatchCount::from(1))
+}
+
+/// Write a dataset with `versions` versions of 10 rows each.
+async fn write_versions(uri: &str, versions: usize, enable_v2_manifest_paths: bool) -> Dataset {
+    let mut ds = Dataset::write(
+        gen_rows(),
+        uri,
+        Some(WriteParams {
+            enable_v2_manifest_paths,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    for _ in 1..versions {
+        ds.append(
+            gen_rows(),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                enable_v2_manifest_paths,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+    }
+    ds
+}
+
 #[tokio::test]
 async fn test_inline_transaction() {
     use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
@@ -382,6 +415,197 @@ async fn test_inline_transaction() {
     assert!(ds_new.manifest.transaction_file.is_some());
     let read_tx = ds_new.read_transaction().await.unwrap().unwrap();
     assert_eq!(read_tx, tx);
+
+    // The direct read takes the same external-file fallback.
+    let version_transaction = ds_new
+        .read_version_transaction(location.version)
+        .await
+        .unwrap();
+    assert_eq!(version_transaction.transaction, Some(tx));
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_does_not_populate_caches() {
+    use lance_index::IndexType;
+    use lance_index::scalar::ScalarIndexParams;
+
+    let test_uri = TempStrDir::default();
+    let mut dataset = write_versions(&test_uri, 1, true).await;
+    // Index the table so historical manifests carry an IndexSection that a
+    // caching read path would decode.
+    dataset
+        .create_index(
+            &["key"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap(); // version 2
+    for _ in 0..18 {
+        dataset
+            .append(
+                gen_rows(),
+                Some(WriteParams {
+                    mode: WriteMode::Append,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap();
+    }
+    let latest_version = dataset.version().version;
+    assert_eq!(latest_version, 20);
+
+    // Fresh session so any cache insertion by the API under test shows as growth.
+    let session = Arc::new(Session::default());
+    let dataset = DatasetBuilder::from_uri(&test_uri)
+        .with_session(session.clone())
+        .load()
+        .await
+        .unwrap();
+
+    let metadata_stats_before = session.metadata_cache_stats().await;
+    let index_stats_before = session.index_cache_stats().await;
+
+    let mut actual = Vec::with_capacity(latest_version as usize);
+    for version in 1..=latest_version {
+        let version_transaction = dataset.read_version_transaction(version).await.unwrap();
+        assert_eq!(version_transaction.version, version);
+        actual.push(version_transaction);
+    }
+
+    let metadata_stats_after = session.metadata_cache_stats().await;
+    let index_stats_after = session.index_cache_stats().await;
+    assert_eq!(
+        metadata_stats_after.num_entries,
+        metadata_stats_before.num_entries
+    );
+    assert_eq!(
+        metadata_stats_after.size_bytes,
+        metadata_stats_before.size_bytes
+    );
+    assert_eq!(
+        index_stats_after.num_entries,
+        index_stats_before.num_entries
+    );
+    assert_eq!(index_stats_after.size_bytes, index_stats_before.size_bytes);
+
+    // Results match a full checkout.
+    for version_transaction in &actual {
+        let checked_out = dataset
+            .checkout_version(version_transaction.version)
+            .await
+            .unwrap();
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+        assert_eq!(
+            version_transaction.timestamp,
+            checked_out.version().timestamp
+        );
+        assert!(version_transaction.transaction.is_some());
+    }
+
+    // A missing (e.g. cleaned up) version errors as DatasetNotFound, matching
+    // the historical checkout_version-based contract of the public API.
+    let err = dataset.read_version_transaction(9999).await.unwrap_err();
+    assert!(
+        matches!(err, crate::Error::DatasetNotFound { .. }),
+        "expected DatasetNotFound for a missing version, got {err:?}"
+    );
+}
+
+#[tokio::test]
+async fn test_read_transaction_recovers_from_stale_manifest_size() {
+    let test_uri = TempStrDir::default();
+    let ds = write_versions(&test_uri, 1, true).await;
+    let manifest = ds.manifest().clone();
+    // Only meaningful for the inline path; a plain write inlines the transaction.
+    assert!(manifest.transaction_section.is_some());
+
+    // A size at/under the transaction offset makes the first read_message fail
+    // "file size is too small"; only the retry at the true size can recover.
+    let mut stale = ds.manifest_location().clone();
+    stale.size = Some(1);
+    let recovered = ds
+        .read_transaction_from_storage(&manifest, &stale)
+        .await
+        .unwrap();
+    assert_eq!(recovered, ds.read_transaction().await.unwrap());
+    assert!(recovered.is_some());
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_v1_manifest_naming() {
+    let test_uri = TempStrDir::default();
+    let ds = write_versions(&test_uri, 3, false).await;
+    assert_eq!(
+        ds.manifest_location().naming_scheme,
+        ManifestNamingScheme::V1
+    );
+
+    for version in 1..=3 {
+        let version_transaction = ds.read_version_transaction(version).await.unwrap();
+        let checked_out = ds.checkout_version(version).await.unwrap();
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+        assert_eq!(
+            version_transaction.timestamp,
+            checked_out.version().timestamp
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_on_branch() {
+    let test_uri = TempStrDir::default();
+    let mut main_ds = write_versions(&test_uri, 1, true).await;
+    let branch_ds = main_ds.create_branch("dev", 1, None).await.unwrap();
+
+    // Commit on the branch.
+    let branch_ds = Dataset::write(
+        gen_rows(),
+        branch_ds.uri(),
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(branch_ds.manifest().branch.as_deref(), Some("dev"));
+
+    // Versions resolve against the branch chain and match a full checkout.
+    for version in branch_ds.versions().await.unwrap() {
+        let version_transaction = branch_ds
+            .read_version_transaction(version.version)
+            .await
+            .unwrap();
+        assert_eq!(version_transaction.version, version.version);
+        assert_eq!(version_transaction.timestamp, version.timestamp);
+        let checked_out = branch_ds.checkout_version(version.version).await.unwrap();
+        assert_eq!(checked_out.manifest().branch.as_deref(), Some("dev"));
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+    }
+
+    // The append on the branch is the branch's own transaction.
+    let latest = branch_ds.version().version;
+    let version_transaction = branch_ds.read_version_transaction(latest).await.unwrap();
+    assert!(matches!(
+        version_transaction.transaction,
+        Some(Transaction {
+            operation: Operation::Append { .. },
+            ..
+        })
+    ));
 }
 
 #[tokio::test]
diff --git a/rust/lance/src/dataset/tests/mod.rs b/rust/lance/src/dataset/tests/mod.rs
index ecc64587b0c..76713abb8d1 100644
--- a/rust/lance/src/dataset/tests/mod.rs
+++ b/rust/lance/src/dataset/tests/mod.rs
@@ -11,6 +11,7 @@ mod dataset_index;
 mod dataset_io;
 mod dataset_merge_update;
 mod dataset_migrations;
+mod dataset_overlay_index_masking;
 mod dataset_scanner;
 mod dataset_schema_evolution;
 mod dataset_transactions;
diff --git a/rust/lance/src/dataset/transaction.rs b/rust/lance/src/dataset/transaction.rs
index 3261f9300c4..34258c83f96 100644
--- a/rust/lance/src/dataset/transaction.rs
+++ b/rust/lance/src/dataset/transaction.rs
@@ -15,7 +15,7 @@
 use super::ManifestWriteConfig;
 use super::write::merge_insert::inserted_rows::KeyExistenceFilter;
 use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows};
-use crate::index::mem_wal::update_mem_wal_index_merged_generations;
+use crate::index::mem_wal::update_mem_wal_index_compacted_sstables;
 use crate::utils::temporal::timestamp_to_nanos;
 use lance_core::datatypes::{
     LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY,
@@ -24,7 +24,7 @@ use lance_core::datatypes::{
 use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, datatypes::Schema};
 use lance_file::{datatypes::Fields, version::LanceFileVersion};
-use lance_index::mem_wal::MergedGeneration;
+use lance_index::mem_wal::CompactedSsTable;
 use lance_index::{frag_reuse::FRAG_REUSE_INDEX_NAME, is_system_index};
 use lance_io::object_store::ObjectStore;
 use lance_table::feature_flags::{FLAG_STABLE_ROW_IDS, apply_feature_flags};
@@ -32,7 +32,8 @@ use lance_table::rowids::read_row_ids;
 use lance_table::{
     format::{
         BasePath, DataFile, DataStorageFormat, Fragment, IndexFile, IndexMetadata, Manifest,
-        RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, pb,
+        RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta,
+        overlay::DataOverlayFile, pb,
     },
     io::{
         commit::CommitHandler,
@@ -258,6 +259,17 @@ pub struct Transaction {
 #[derive(Debug, Clone, DeepSizeOf, PartialEq)]
 pub struct DataReplacementGroup(pub u64, pub DataFile);
 
+/// Overlay files to append to a single fragment, in order (the last entry is
+/// newest). The overlays are appended to the fragment's existing `overlays`
+/// list rather than replacing it, so overlays written by concurrent commits are
+/// preserved. Each overlay's `committed_version` is stamped to the new dataset
+/// version at commit time (re-stamped on retry).
+#[derive(Debug, Clone, DeepSizeOf, PartialEq)]
+pub struct DataOverlayGroup {
+    pub fragment_id: u64,
+    pub overlays: Vec<DataOverlayFile>,
+}
+
 /// An entry for a map update. If value is None, the key will be removed from the map.
 #[derive(Debug, Clone, DeepSizeOf, PartialEq)]
 pub struct UpdateMapEntry {
@@ -367,6 +379,11 @@ pub enum Operation {
     DataReplacement {
         replacements: Vec<DataReplacementGroup>,
     },
+    /// Attach overlay files to fragments, supplying new values for a subset of
+    /// `(physical offset, field)` cells without rewriting the fragments' base
+    /// data files. See [`DataOverlayFile`] and the Data Overlay Files
+    /// specification for resolution, coverage, and versioning rules.
+    DataOverlay { groups: Vec<DataOverlayGroup> },
     /// Merge a new column in
     /// 'fragments' is the final fragments include all data files, the new fragments must align with old ones at rows.
     /// 'schema' is not forced to include existed columns, which means we could use Merge to drop column data
@@ -407,8 +424,8 @@ pub enum Operation {
         new_fragments: Vec<Fragment>,
         /// The fields that have been modified
         fields_modified: Vec<u32>,
-        /// List of MemWAL region generations to mark as merged after this transaction
-        merged_generations: Vec<MergedGeneration>,
+        /// MemWAL SSTables to mark as compacted after this transaction.
+        compacted_sstables: Vec<CompactedSsTable>,
         /// The fields that used to judge whether to preserve the new frag's id into
         /// the frag bitmap of the specified indices.
         fields_for_preserving_frag_bitmap: Vec<u32>,
@@ -432,11 +449,12 @@ pub enum Operation {
         schema_metadata_updates: Option<UpdateMap>,
         field_metadata_updates: HashMap<i32, UpdateMap>,
     },
-    /// Update merged generations in MemWAL index.
+    /// Update SSTable compaction progress in the MemWAL index.
+    ///
     /// This is used during merge-insert to atomically record which
-    /// generations have been merged to the base table.
+    /// SSTables have been compacted into the base table.
     UpdateMemWalState {
-        merged_generations: Vec<MergedGeneration>,
+        compacted_sstables: Vec<CompactedSsTable>,
     },
 
     /// Clone a dataset.
@@ -499,6 +517,7 @@ impl std::fmt::Display for Operation {
             Self::Project { .. } => write!(f, "Project"),
             Self::UpdateConfig { .. } => write!(f, "UpdateConfig"),
             Self::DataReplacement { .. } => write!(f, "DataReplacement"),
+            Self::DataOverlay { .. } => write!(f, "DataOverlay"),
             Self::Clone { .. } => write!(f, "Clone"),
             Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"),
             Self::UpdateBases { .. } => write!(f, "UpdateBases"),
@@ -631,7 +650,7 @@ impl PartialEq for Operation {
                     updated_fragments: a_updated,
                     new_fragments: a_new,
                     fields_modified: a_fields,
-                    merged_generations: a_merged_generations,
+                    compacted_sstables: a_compacted_sstables,
                     fields_for_preserving_frag_bitmap: a_fields_for_preserving_frag_bitmap,
                     update_mode: a_update_mode,
                     inserted_rows_filter: a_inserted_rows_filter,
@@ -642,7 +661,7 @@ impl PartialEq for Operation {
                     updated_fragments: b_updated,
                     new_fragments: b_new,
                     fields_modified: b_fields,
-                    merged_generations: b_merged_generations,
+                    compacted_sstables: b_compacted_sstables,
                     fields_for_preserving_frag_bitmap: b_fields_for_preserving_frag_bitmap,
                     update_mode: b_update_mode,
                     inserted_rows_filter: b_inserted_rows_filter,
@@ -653,7 +672,7 @@ impl PartialEq for Operation {
                     && compare_vec(a_updated, b_updated)
                     && compare_vec(a_new, b_new)
                     && compare_vec(a_fields, b_fields)
-                    && compare_vec(a_merged_generations, b_merged_generations)
+                    && compare_vec(a_compacted_sstables, b_compacted_sstables)
                     && compare_vec(
                         a_fields_for_preserving_frag_bitmap,
                         b_fields_for_preserving_frag_bitmap,
@@ -1210,12 +1229,12 @@ impl PartialEq for Operation {
             }
             (
                 Self::UpdateMemWalState {
-                    merged_generations: a_merged,
+                    compacted_sstables: a_compacted,
                 },
                 Self::UpdateMemWalState {
-                    merged_generations: b_merged,
+                    compacted_sstables: b_compacted,
                 },
-            ) => compare_vec(a_merged, b_merged),
+            ) => compare_vec(a_compacted, b_compacted),
             (Self::Clone { .. }, Self::Append { .. }) => {
                 std::mem::discriminant(self) == std::mem::discriminant(other)
             }
@@ -1345,6 +1364,8 @@ impl PartialEq for Operation {
             (Self::Clone { .. }, Self::UpdateBases { .. }) => {
                 std::mem::discriminant(self) == std::mem::discriminant(other)
             }
+            (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b),
+            (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false,
         }
     }
 }
@@ -1521,6 +1542,7 @@ impl Operation {
             Self::Project { .. } => "Project",
             Self::UpdateConfig { .. } => "UpdateConfig",
             Self::DataReplacement { .. } => "DataReplacement",
+            Self::DataOverlay { .. } => "DataOverlay",
             Self::UpdateMemWalState { .. } => "UpdateMemWalState",
             Self::Clone { .. } => "Clone",
             Self::UpdateBases { .. } => "UpdateBases",
@@ -1861,6 +1883,8 @@ impl Transaction {
                     ))
                 });
 
+        let new_version = current_manifest.map_or(1, |m| m.version + 1);
+
         match &self.operation {
             Operation::Clone { .. } => {
                 return Err(Error::internal(
@@ -1875,7 +1899,6 @@ impl Transaction {
                 if let Some(next_row_id) = &mut next_row_id {
                     Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?;
                     // Add version metadata for all new fragments
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     for fragment in new_fragments.iter_mut() {
                         let version_meta = build_version_meta(fragment, new_version);
                         fragment.last_updated_at_version_meta = version_meta.clone();
@@ -1906,7 +1929,7 @@ impl Transaction {
                 updated_fragments,
                 new_fragments,
                 fields_modified,
-                merged_generations,
+                compacted_sstables,
                 fields_for_preserving_frag_bitmap,
                 update_mode,
                 updated_fragment_offsets,
@@ -1923,7 +1946,20 @@ impl Transaction {
                             return None;
                         }
                         if let Some(updated) = updated_fragments.iter().find(|uf| uf.id == f.id) {
-                            Some(updated.clone())
+                            let mut updated = updated.clone();
+                            // Carry forward the fragment's current overlays (which
+                            // may include ones added by a concurrent commit). An
+                            // in-place column rewrite then tombstones the overlaid
+                            // fields it rewrote, since the fresh base values
+                            // supersede them.
+                            updated.overlays = f.overlays.clone();
+                            if matches!(update_mode, Some(RewriteColumns)) {
+                                lance_table::format::overlay::tombstone_overlay_fields(
+                                    &mut updated.overlays,
+                                    fields_modified,
+                                );
+                            }
+                            Some(updated)
                         } else {
                             Some(f.clone())
                         }
@@ -1945,7 +1981,6 @@ impl Transaction {
                     && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets
                     && !off_map.is_empty()
                 {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     let prev_version = current_manifest.map(|m| m.version).unwrap_or(0);
                     for fragment in final_fragments.iter_mut() {
                         let Some(bitmap) = off_map.get(&fragment.id) else {
@@ -1988,7 +2023,6 @@ impl Transaction {
                 }
 
                 if next_row_id.is_some() {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     resolve_update_version_metadata(
                         existing_fragments,
                         new_fragments.as_mut_slice(),
@@ -2030,11 +2064,11 @@ impl Transaction {
                 final_fragments.extend(new_fragments);
                 Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments);
 
-                if !merged_generations.is_empty() {
-                    update_mem_wal_index_merged_generations(
+                if !compacted_sstables.is_empty() {
+                    update_mem_wal_index_compacted_sstables(
                         &mut final_indices,
-                        current_manifest.map_or(1, |m| m.version + 1),
-                        merged_generations.clone(),
+                        new_version,
+                        compacted_sstables.clone(),
                     )?;
                 }
             }
@@ -2045,7 +2079,6 @@ impl Transaction {
                 if let Some(next_row_id) = &mut next_row_id {
                     Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?;
                     // Add version metadata for all new fragments
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     for fragment in new_fragments.iter_mut() {
                         let version_meta = build_version_meta(fragment, new_version);
                         fragment.last_updated_at_version_meta = version_meta.clone();
@@ -2083,6 +2116,12 @@ impl Transaction {
                     Self::handle_rewrite_indices(&mut final_indices, rewritten_indices, groups)?;
                 }
 
+                // A full compaction materializes a fragment's overlays into fresh
+                // base data. Any index older than one of those overlays was built on
+                // the pre-overlay values, so drop the rewritten fragment from its
+                // coverage to keep it from serving stale values.
+                Self::prune_overlay_stale_fields_from_indices(&mut final_indices, groups);
+
                 if let Some(frag_reuse_index) = frag_reuse_index {
                     final_indices.retain(|idx| idx.name != frag_reuse_index.name);
                     final_indices.push(frag_reuse_index.clone());
@@ -2114,7 +2153,6 @@ impl Transaction {
                 let existing_fragments = maybe_existing_fragments?;
                 let mut merged_fragments = fragments.clone();
                 if next_row_id.is_some() {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     let prev_by_id: HashMap<u64, &Fragment> =
                         existing_fragments.iter().map(|f| (f.id, f)).collect();
                     for fragment in merged_fragments.iter_mut() {
@@ -2274,6 +2312,15 @@ impl Transaction {
                             "Expected to modify the fragment but no changes were made. This means the new data files does not align with any exiting datafiles. Please check if the schema of the new data files matches the schema of the old data files including the file major and minor versions",
                         ));
                     }
+
+                    // New base values for these fields supersede any overlay
+                    // still shadowing them; tombstone the overlaid fields so the
+                    // replacement is not silently masked.
+                    lance_table::format::overlay::tombstone_overlay_fields(
+                        &mut new_frag.overlays,
+                        &replaced_fields,
+                    );
+
                     final_fragments.push(new_frag);
                 }
 
@@ -2305,11 +2352,58 @@ impl Transaction {
                     &replaced_fields,
                 );
             }
-            Operation::UpdateMemWalState { merged_generations } => {
-                update_mem_wal_index_merged_generations(
+            Operation::DataOverlay { groups } => {
+                // Stamp each overlay with the version this commit is producing.
+                // build_manifest re-runs on every retry with an updated
+                // current_manifest, so this is naturally re-stamped on retry.
+                let new_version = current_manifest.map_or(1, |m| m.version + 1);
+
+                let existing_fragments = maybe_existing_fragments?;
+                // Multiple groups may target the same fragment; merge them in
+                // order rather than letting a HashMap collapse drop all but the
+                // last group's overlays.
+                let mut overlays_by_fragment: HashMap<u64, Vec<&DataOverlayFile>> = HashMap::new();
+                for group in groups {
+                    overlays_by_fragment
+                        .entry(group.fragment_id)
+                        .or_default()
+                        .extend(group.overlays.iter());
+                }
+
+                // Every group must target an existing fragment. Build a set of
+                // existing ids once so this is O(groups + fragments) rather than
+                // O(groups * fragments).
+                let existing_fragment_ids: HashSet<u64> =
+                    existing_fragments.iter().map(|f| f.id).collect();
+                for fragment_id in overlays_by_fragment.keys() {
+                    if !existing_fragment_ids.contains(fragment_id) {
+                        return Err(Error::invalid_input(format!(
+                            "DataOverlay targets fragment {fragment_id}, which does not exist"
+                        )));
+                    }
+                }
+
+                for fragment in existing_fragments {
+                    let mut fragment = fragment.clone();
+                    if let Some(new_overlays) = overlays_by_fragment.get(&fragment.id) {
+                        // Appended (not replaced) so concurrently-written overlays
+                        // survive; later entries are newer.
+                        fragment
+                            .overlays
+                            .extend(new_overlays.iter().map(|&overlay| {
+                                let mut overlay = overlay.clone();
+                                overlay.committed_version = new_version;
+                                overlay
+                            }));
+                    }
+                    final_fragments.push(fragment);
+                }
+            }
+            Operation::UpdateMemWalState { compacted_sstables } => {
+                update_mem_wal_index_compacted_sstables(
                     &mut final_indices,
-                    current_manifest.map_or(1, |m| m.version + 1),
-                    merged_generations.clone(),
+                    new_version,
+                    compacted_sstables.clone(),
                 )?;
             }
             Operation::UpdateBases { .. } => {
@@ -2325,6 +2419,15 @@ impl Transaction {
         // Clean up data files that only contain tombstoned fields
         Self::remove_tombstoned_data_files(&mut final_fragments);
 
+        // Enforce the newest-last overlay ordering invariant at the write
+        // boundary. Load normalizes with a sort; this rejects any commit path
+        // that assembled a fragment's overlays out of order.
+        for fragment in &final_fragments {
+            if !fragment.overlays.is_empty() {
+                lance_table::format::overlay::verify_overlays_newest_last(&fragment.overlays)?;
+            }
+        }
+
         let user_requested_version = match (&config.storage_format, config.use_legacy_format) {
             (Some(storage_format), _) => Some(storage_format.lance_file_version()?),
             (None, Some(true)) => Some(LanceFileVersion::Legacy),
@@ -2682,6 +2785,56 @@ impl Transaction {
         }
     }
 
+    /// After a `Rewrite` fully compacts a fragment, its data overlays are baked
+    /// into the new fragment's base data. An index built *before* one of those
+    /// overlays (`overlay.committed_version > index.dataset_version`) indexed the
+    /// stale pre-overlay values -- and unlike a live overlay, the compacted
+    /// fragment no longer signals that staleness to the query path. Drop each
+    /// rewritten (new) fragment from the coverage of any index covering a field
+    /// such an overlay supplied, so those rows fall back to a flat scan.
+    fn prune_overlay_stale_fields_from_indices(
+        indices: &mut [IndexMetadata],
+        groups: &[RewriteGroup],
+    ) {
+        for group in groups {
+            // field id -> newest overlay committed_version supplying that field
+            let mut overlaid_field_versions: HashMap<i32, u64> = HashMap::new();
+            for old_frag in &group.old_fragments {
+                for overlay in &old_frag.overlays {
+                    for &field_id in overlay.data_file.fields.iter() {
+                        if field_id < 0 {
+                            // Tombstoned (obsolete) overlay field: supplies nothing.
+                            continue;
+                        }
+                        let entry = overlaid_field_versions.entry(field_id).or_insert(0);
+                        *entry = (*entry).max(overlay.committed_version);
+                    }
+                }
+            }
+            if overlaid_field_versions.is_empty() {
+                continue;
+            }
+
+            let new_fragment_ids = group
+                .new_fragments
+                .iter()
+                .map(|f| f.id as u32)
+                .collect::<Vec<_>>();
+            for index in indices.iter_mut() {
+                let is_stale = index.fields.iter().any(|field_id| {
+                    overlaid_field_versions
+                        .get(field_id)
+                        .is_some_and(|&overlay_version| overlay_version > index.dataset_version)
+                });
+                if is_stale && let Some(fragment_bitmap) = &mut index.fragment_bitmap {
+                    for new_id in &new_fragment_ids {
+                        fragment_bitmap.remove(*new_id);
+                    }
+                }
+            }
+        }
+    }
+
     fn is_vector_index(index: &IndexMetadata) -> bool {
         if let Some(details) = &index.index_details {
             details.type_url.ends_with("VectorIndexDetails")
@@ -3066,6 +3219,34 @@ impl TryFrom<pb::transaction::DataReplacementGroup> for DataReplacementGroup {
     }
 }
 
+impl From<&DataOverlayGroup> for pb::transaction::DataOverlayGroup {
+    fn from(group: &DataOverlayGroup) -> Self {
+        Self {
+            fragment_id: group.fragment_id,
+            overlays: group
+                .overlays
+                .iter()
+                .map(pb::DataOverlayFile::from)
+                .collect(),
+        }
+    }
+}
+
+impl TryFrom<pb::transaction::DataOverlayGroup> for DataOverlayGroup {
+    type Error = Error;
+
+    fn try_from(message: pb::transaction::DataOverlayGroup) -> Result<Self> {
+        Ok(Self {
+            fragment_id: message.fragment_id,
+            overlays: message
+                .overlays
+                .into_iter()
+                .map(DataOverlayFile::try_from)
+                .collect::<Result<Vec<_>>>()?,
+        })
+    }
+}
+
 impl TryFrom<pb::Transaction> for Transaction {
     type Error = Error;
 
@@ -3122,7 +3303,7 @@ impl TryFrom<pb::Transaction> for Transaction {
                         .into_iter()
                         .map(Fragment::try_from)
                         .collect::<Result<Vec<_>>>()?,
-                    schema: Schema::from(&Fields(schema)),
+                    schema: Schema::try_from(&Fields(schema))?,
                     config_upsert_values: config_upsert_option,
                     initial_bases: if initial_bases.is_empty() {
                         None
@@ -3190,7 +3371,7 @@ impl TryFrom<pb::Transaction> for Transaction {
                     .into_iter()
                     .map(Fragment::try_from)
                     .collect::<Result<Vec<_>>>()?,
-                schema: Schema::from(&Fields(schema)),
+                schema: Schema::try_from(&Fields(schema))?,
             },
             Some(pb::transaction::Operation::Restore(pb::transaction::Restore { version })) => {
                 Operation::Restore { version }
@@ -3200,7 +3381,7 @@ impl TryFrom<pb::Transaction> for Transaction {
                 updated_fragments,
                 new_fragments,
                 fields_modified,
-                merged_generations,
+                compacted_sstables,
                 fields_for_preserving_frag_bitmap,
                 update_mode,
                 inserted_rows,
@@ -3216,9 +3397,9 @@ impl TryFrom<pb::Transaction> for Transaction {
                     .map(Fragment::try_from)
                     .collect::<Result<Vec<_>>>()?,
                 fields_modified,
-                merged_generations: merged_generations
+                compacted_sstables: compacted_sstables
                     .into_iter()
-                    .map(|m| MergedGeneration::try_from(m).unwrap())
+                    .map(|m| CompactedSsTable::try_from(m).unwrap())
                     .collect(),
                 fields_for_preserving_frag_bitmap,
                 update_mode: match update_mode {
@@ -3244,7 +3425,7 @@ impl TryFrom<pb::Transaction> for Transaction {
             },
             Some(pb::transaction::Operation::Project(pb::transaction::Project { schema })) => {
                 Operation::Project {
-                    schema: Schema::from(&Fields(schema)),
+                    schema: Schema::try_from(&Fields(schema))?,
                 }
             }
             Some(pb::transaction::Operation::UpdateConfig(update_config)) => {
@@ -3336,11 +3517,11 @@ impl TryFrom<pb::Transaction> for Transaction {
                     .collect::<Result<Vec<_>>>()?,
             },
             Some(pb::transaction::Operation::UpdateMemWalState(
-                pb::transaction::UpdateMemWalState { merged_generations },
+                pb::transaction::UpdateMemWalState { compacted_sstables },
             )) => Operation::UpdateMemWalState {
-                merged_generations: merged_generations
+                compacted_sstables: compacted_sstables
                     .into_iter()
-                    .map(|m| MergedGeneration::try_from(m).unwrap())
+                    .map(|m| CompactedSsTable::try_from(m).unwrap())
                     .collect(),
             },
             Some(pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases {
@@ -3348,6 +3529,14 @@ impl TryFrom<pb::Transaction> for Transaction {
             })) => Operation::UpdateBases {
                 new_bases: new_bases.into_iter().map(BasePath::from).collect(),
             },
+            Some(pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay {
+                groups,
+            })) => Operation::DataOverlay {
+                groups: groups
+                    .into_iter()
+                    .map(DataOverlayGroup::try_from)
+                    .collect::<Result<Vec<_>>>()?,
+            },
             None => {
                 return Err(Error::internal(
                     "Transaction message did not contain an operation".to_string(),
@@ -3539,7 +3728,7 @@ impl From<&Transaction> for pb::Transaction {
                 updated_fragments,
                 new_fragments,
                 fields_modified,
-                merged_generations,
+                compacted_sstables,
                 fields_for_preserving_frag_bitmap,
                 update_mode,
                 inserted_rows_filter,
@@ -3552,9 +3741,9 @@ impl From<&Transaction> for pb::Transaction {
                     .collect(),
                 new_fragments: new_fragments.iter().map(pb::DataFragment::from).collect(),
                 fields_modified: fields_modified.clone(),
-                merged_generations: merged_generations
+                compacted_sstables: compacted_sstables
                     .iter()
-                    .map(pb::MergedGeneration::from)
+                    .map(pb::CompactedSsTable::from)
                     .collect(),
                 fields_for_preserving_frag_bitmap: fields_for_preserving_frag_bitmap.clone(),
                 update_mode: update_mode
@@ -3618,11 +3807,19 @@ impl From<&Transaction> for pb::Transaction {
                         .collect(),
                 })
             }
-            Operation::UpdateMemWalState { merged_generations } => {
+            Operation::DataOverlay { groups } => {
+                pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay {
+                    groups: groups
+                        .iter()
+                        .map(pb::transaction::DataOverlayGroup::from)
+                        .collect(),
+                })
+            }
+            Operation::UpdateMemWalState { compacted_sstables } => {
                 pb::transaction::Operation::UpdateMemWalState(pb::transaction::UpdateMemWalState {
-                    merged_generations: merged_generations
+                    compacted_sstables: compacted_sstables
                         .iter()
-                        .map(pb::MergedGeneration::from)
+                        .map(pb::CompactedSsTable::from)
                         .collect::<Vec<_>>(),
                 })
             }
@@ -3896,6 +4093,7 @@ mod tests {
     use lance_core::{ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_LAST_UPDATED_AT_VERSION};
     use lance_file::version::LanceFileVersion;
     use lance_io::utils::CachedFileSize;
+    use lance_table::format::overlay::OverlayCoverage;
     use lance_table::format::{
         RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta,
     };
@@ -4207,6 +4405,7 @@ mod tests {
             physical_rows: Some(100),
             row_id_meta: None,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4239,6 +4438,7 @@ mod tests {
             physical_rows: Some(50),
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4271,6 +4471,7 @@ mod tests {
             physical_rows: Some(50), // More physical rows than existing row IDs
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4306,6 +4507,7 @@ mod tests {
             physical_rows: Some(50), // Less physical rows than existing row IDs
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4334,6 +4536,7 @@ mod tests {
                 physical_rows: Some(30), // No existing row IDs
                 row_id_meta: None,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 last_updated_at_version_meta: None,
                 created_at_version_meta: None,
@@ -4343,6 +4546,7 @@ mod tests {
                 physical_rows: Some(25), // Partial existing row IDs
                 row_id_meta: Some(RowIdMeta::Inline(serialized)),
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 last_updated_at_version_meta: None,
                 created_at_version_meta: None,
@@ -4387,6 +4591,7 @@ mod tests {
             physical_rows: None,
             row_id_meta: None,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4894,6 +5099,7 @@ mod tests {
         let fragment = Fragment {
             id: 1,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -4912,7 +5118,7 @@ mod tests {
                 updated_fragments: vec![fragment],
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations: vec![],
+                compacted_sstables: vec![],
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: Some(UpdateMode::RewriteColumns),
                 inserted_rows_filter: None,
@@ -5065,7 +5271,7 @@ mod tests {
             updated_fragments: vec![u.fragment],
             new_fragments: vec![],
             fields_modified: u.fields_modified,
-            merged_generations: Vec::new(),
+            compacted_sstables: Vec::new(),
             fields_for_preserving_frag_bitmap: vec![],
             update_mode: Some(UpdateMode::RewriteColumns),
             inserted_rows_filter: None,
@@ -5163,6 +5369,7 @@ mod tests {
                 None,
             )],
             physical_rows: Some(10),
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             last_updated_at_version_meta: None,
@@ -5205,7 +5412,7 @@ mod tests {
                 updated_fragments: vec![],
                 new_fragments,
                 fields_modified: vec![],
-                merged_generations: vec![],
+                compacted_sstables: vec![],
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -5254,6 +5461,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![mk_file("before.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -5326,6 +5534,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![data_file.clone()],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: row_id_meta.clone(),
             physical_rows: Some(5),
@@ -5345,6 +5554,7 @@ mod tests {
         let merged_fragment = Fragment {
             id: 0,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -5394,6 +5604,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![mk_file("before.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(5),
@@ -5458,6 +5669,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 0,
             files: vec![mk_file("existing.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0))),
             physical_rows: Some(3),
@@ -5480,6 +5692,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 1,
             files: vec![mk_file("new.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1))),
             physical_rows: Some(4),
@@ -5542,6 +5755,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(3),
@@ -5555,6 +5769,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5597,6 +5812,7 @@ mod tests {
             Fragment {
                 id: 1,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq))),
                 physical_rows: Some(2),
@@ -5608,6 +5824,7 @@ mod tests {
             Fragment {
                 id: 2,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq))),
                 physical_rows: Some(3),
@@ -5623,6 +5840,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5664,6 +5882,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5678,6 +5897,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5722,6 +5942,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5735,6 +5956,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 20,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(4),
@@ -5768,6 +5990,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5779,6 +6002,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(1),
@@ -5807,6 +6031,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5817,6 +6042,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(3),
@@ -5847,6 +6073,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5860,6 +6087,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(1),
@@ -5901,6 +6129,7 @@ mod tests {
         let in_range_frag = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq))),
             physical_rows: Some(2),
@@ -5921,6 +6150,7 @@ mod tests {
         let out_of_range_frag = Fragment {
             id: 2,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq))),
             physical_rows: Some(2),
@@ -5935,6 +6165,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5973,6 +6204,7 @@ mod tests {
         let existing = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq))),
             physical_rows: Some(3),
@@ -5985,6 +6217,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -6033,6 +6266,7 @@ mod tests {
         let src_frag = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq))),
             physical_rows: Some(100),
@@ -6047,6 +6281,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(100),
@@ -6094,6 +6329,7 @@ mod tests {
             Fragment {
                 id: 1,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a))),
                 physical_rows: Some(3),
@@ -6105,6 +6341,7 @@ mod tests {
             Fragment {
                 id: 2,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b))),
                 physical_rows: Some(3),
@@ -6120,6 +6357,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -6186,4 +6424,311 @@ mod tests {
         assert!(!left.modifies_same_metadata(&different_key));
         assert!(left.modifies_same_metadata(&replace));
     }
+
+    #[test]
+    fn test_data_overlay_operation_roundtrips() {
+        // A DataOverlay operation survives the protobuf round-trip, preserving
+        // the target fragment, the overlay's coverage, and its committed_version.
+        let mut bitmap = roaring::RoaringBitmap::new();
+        bitmap.insert(1);
+        bitmap.insert(4);
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(bitmap.clone()),
+            committed_version: 6,
+        };
+        let pb_overlay = pb::DataOverlayFile::from(&overlay);
+
+        let message = pb::Transaction {
+            read_version: 1,
+            uuid: Uuid::new_v4().to_string(),
+            operation: Some(pb::transaction::Operation::DataOverlay(
+                pb::transaction::DataOverlay {
+                    groups: vec![pb::transaction::DataOverlayGroup {
+                        fragment_id: 7,
+                        overlays: vec![pb_overlay],
+                    }],
+                },
+            )),
+            ..Default::default()
+        };
+
+        let txn = Transaction::try_from(message).unwrap();
+        match txn.operation {
+            Operation::DataOverlay { groups } => {
+                assert_eq!(groups.len(), 1);
+                assert_eq!(groups[0].fragment_id, 7);
+                assert_eq!(groups[0].overlays.len(), 1);
+                assert_eq!(groups[0].overlays[0].committed_version, 6);
+                assert_eq!(
+                    *groups[0].overlays[0].coverage_for_field(0).unwrap(),
+                    bitmap
+                );
+            }
+            other => panic!("expected DataOverlay, got {other:?}"),
+        }
+    }
+
+    fn overlay_with_field(field: i32, committed_version: u64) -> DataOverlayFile {
+        DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None),
+            coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])),
+            committed_version,
+        }
+    }
+
+    #[test]
+    fn test_prune_overlay_stale_fields_from_indices() {
+        // Fragment 0 carried an overlay on field 1 committed at v5, and was
+        // fully compacted into new fragment 7.
+        let mut old_frag = Fragment::new(0);
+        old_frag.overlays = vec![overlay_with_field(1, 5)];
+        let groups = vec![RewriteGroup {
+            old_fragments: vec![old_frag],
+            new_fragments: vec![Fragment::new(7)],
+        }];
+
+        // Post-remap state: every index already covers the new fragment (7).
+        let covering = || Some(RoaringBitmap::from_iter([7u32]));
+        let mut indices = vec![
+            // Stale: covers the overlaid field 1, built (v2) before the overlay.
+            create_test_index("stale", 1, 2, covering(), false),
+            // Not stale: covers field 1 but built at the overlay's version (v5);
+            // `committed_version > dataset_version` is false at equality.
+            create_test_index("fresh", 1, 5, covering(), false),
+            // Unrelated: covers field 2, which the overlay never touched.
+            create_test_index("unrelated", 2, 2, covering(), false),
+        ];
+
+        Transaction::prune_overlay_stale_fields_from_indices(&mut indices, &groups);
+
+        assert!(
+            !indices[0].fragment_bitmap.as_ref().unwrap().contains(7),
+            "stale index must drop the rewritten fragment from its coverage"
+        );
+        assert!(
+            indices[1].fragment_bitmap.as_ref().unwrap().contains(7),
+            "an index built at/after the overlay is not stale"
+        );
+        assert!(
+            indices[2].fragment_bitmap.as_ref().unwrap().contains(7),
+            "an index on an un-overlaid field is unaffected"
+        );
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_multi_fragment() {
+        // Overlays targeting two distinct fragments are each applied and stamped.
+        // A targeted fragment already carrying an overlay (committed at v3) gets
+        // the new overlay appended and stamped while its existing overlay is
+        // preserved, and a fragment the operation does not target is passed
+        // through with its existing overlays untouched.
+        let mut frag0 = Fragment::new(0);
+        frag0.overlays = vec![overlay_with_field(5, 3)]; // targeted, pre-existing at v3
+        let frag1 = Fragment::new(1);
+        let mut frag2 = Fragment::new(2);
+        frag2.overlays = vec![overlay_with_field(9, 3)]; // untargeted, committed at v3
+        let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]);
+        let mut manifest = Manifest::new(
+            LanceSchema::try_from(&schema).unwrap(),
+            Arc::new(vec![frag0, frag1, frag2]),
+            lance_table::format::DataStorageFormat::new(LanceFileVersion::V2_0),
+            HashMap::new(),
+        );
+        // The pre-existing overlays were committed at v3, so the current
+        // manifest must be at least that version; the new commit then stamps
+        // its overlay at v4, keeping the fragment's overlays newest-last.
+        manifest.version = 3;
+
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(1, 0)],
+                    },
+                    DataOverlayGroup {
+                        fragment_id: 1,
+                        overlays: vec![overlay_with_field(2, 0)],
+                    },
+                ],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let frag = |id: u64| {
+            result
+                .fragments
+                .iter()
+                .find(|f| f.id == id)
+                .unwrap_or_else(|| panic!("fragment {id} missing from result"))
+        };
+        // The already-overlaid target keeps its v3 overlay and appends the new
+        // one, stamped to the new version.
+        assert_eq!(frag(0).overlays.len(), 2);
+        assert_eq!(frag(0).overlays[0].committed_version, 3);
+        assert_eq!(frag(0).overlays[1].committed_version, result.version);
+        // The fresh target gets its overlay, stamped to the new version.
+        assert_eq!(frag(1).overlays.len(), 1);
+        assert_eq!(frag(1).overlays[0].committed_version, result.version);
+        // The untargeted fragment is unchanged: same overlay, original version.
+        assert_eq!(frag(2).overlays.len(), 1);
+        assert_eq!(frag(2).overlays[0].committed_version, 3);
+        assert!(result.version > manifest.version);
+    }
+
+    #[test]
+    fn test_data_replacement_tombstones_overlaid_fields() {
+        // A DataReplacement writing new base values for field 5 must stop any
+        // overlay from shadowing those cells: field 5 is tombstoned in place
+        // (preserving the overlay's field 3), and an overlay covering only field
+        // 5 is dropped entirely.
+        let mut fragment = Fragment::new(0);
+        fragment.files = vec![
+            DataFile::new_legacy_from_fields("f3.lance", vec![3], None),
+            DataFile::new_legacy_from_fields("f5.lance", vec![5], None),
+        ];
+        fragment.overlays = vec![
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("o35.lance", vec![3, 5], None),
+                coverage: OverlayCoverage::sparse(vec![
+                    roaring::RoaringBitmap::from_iter([0u32]),
+                    roaring::RoaringBitmap::from_iter([0u32]),
+                ]),
+                committed_version: 3,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("o5.lance", vec![5], None),
+                coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])),
+                committed_version: 3,
+            },
+        ];
+
+        let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]);
+        let manifest = Manifest::new(
+            LanceSchema::try_from(&schema).unwrap(),
+            Arc::new(vec![fragment]),
+            lance_table::format::DataStorageFormat::new(LanceFileVersion::V2_0),
+            HashMap::new(),
+        );
+
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataReplacement {
+                replacements: vec![DataReplacementGroup(
+                    0,
+                    DataFile::new_legacy_from_fields("f5-new.lance", vec![5], None),
+                )],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let frag = &result.fragments[0];
+        // The base data file for field 5 was swapped in.
+        assert!(frag.files.iter().any(|f| f.path == "f5-new.lance"));
+        // The [3, 5] overlay keeps field 3 and tombstones field 5; the [5]-only
+        // overlay is dropped.
+        assert_eq!(frag.overlays.len(), 1);
+        assert_eq!(frag.overlays[0].data_file.fields.as_ref(), &[3, -2]);
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_merges_duplicate_groups() {
+        // Two groups targeting the same fragment must both survive (a HashMap
+        // collapse would have dropped the first).
+        let manifest = sample_manifest();
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(1, 0)],
+                    },
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(2, 0)],
+                    },
+                ],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let overlays = &result.fragments[0].overlays;
+        assert_eq!(overlays.len(), 2);
+        assert_eq!(overlays[0].data_file.fields.as_ref(), [1i32].as_slice());
+        assert_eq!(overlays[1].data_file.fields.as_ref(), [2i32].as_slice());
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_rejects_unknown_fragment() {
+        let manifest = sample_manifest();
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![DataOverlayGroup {
+                    fragment_id: 99,
+                    overlays: vec![overlay_with_field(1, 0)],
+                }],
+            },
+            None,
+        );
+        let err = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap_err();
+        assert!(err.to_string().contains("does not exist"), "{err}");
+    }
+
+    #[test]
+    fn test_data_overlay_operation_eq() {
+        let overlay = |field: i32| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id: 0,
+                overlays: vec![overlay_with_field(field, 1)],
+            }],
+        };
+        // Reflexive and value-based (the arm previously returned false for self).
+        assert_eq!(overlay(1), overlay(1));
+        assert_ne!(overlay(1), overlay(2));
+        // Not equal to a different operation kind (previously returned true vs Rewrite).
+        let rewrite = Operation::Rewrite {
+            groups: vec![],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
+        };
+        assert_ne!(overlay(1), rewrite);
+    }
 }
diff --git a/rust/lance/src/dataset/udtf.rs b/rust/lance/src/dataset/udtf.rs
index 75c0388bc24..2144b108859 100644
--- a/rust/lance/src/dataset/udtf.rs
+++ b/rust/lance/src/dataset/udtf.rs
@@ -13,7 +13,6 @@ use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD};
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::parser::from_json;
 use serde_json::Value;
-use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
@@ -61,10 +60,6 @@ impl FtsTableProvider {
 
 #[async_trait]
 impl TableProvider for FtsTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.full_schema.clone()
     }
diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs
index 80afa49ffdd..b9226146e41 100644
--- a/rust/lance/src/dataset/write.rs
+++ b/rust/lance/src/dataset/write.rs
@@ -2,10 +2,10 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use arrow_array::RecordBatch;
+use bytes::Bytes;
 use chrono::TimeDelta;
 use datafusion::physical_plan::SendableRecordBatchStream;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use lance_arrow::{
     ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
     BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
@@ -14,12 +14,9 @@ use lance_arrow::{
 use lance_core::datatypes::{
     NullabilityComparison, OnMissing, OnTypeMismatch, SchemaCompareOptions,
 };
-use lance_core::error::LanceOptionExt;
-use lance_core::utils::tempfile::TempDir;
 use lance_core::utils::tracing::{AUDIT_MODE_CREATE, AUDIT_TYPE_DATA, TRACE_FILE_AUDIT};
 use lance_core::{Error, Result, datatypes::Schema};
 use lance_datafusion::chunker::{break_stream, chunk_stream};
-use lance_datafusion::spill::{SpillReceiver, SpillSender, create_replay_spill};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::previous::writer::{
     FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider,
@@ -29,7 +26,7 @@ use lance_file::writer::{self as current_writer, FileWriterOptions};
 use lance_io::object_store::{
     ObjectStore, ObjectStoreParams, ObjectStoreRegistry, parse_base_scoped_key,
 };
-use lance_table::format::{BasePath, DataFile, Fragment};
+use lance_table::format::{BasePath, DataFile, Fragment, IndexMetadata};
 use lance_table::io::commit::{CommitHandler, commit_handler_from_url};
 use lance_table::io::manifest::ManifestDescribing;
 use object_store::path::Path;
@@ -47,6 +44,8 @@ use crate::dataset::blob::{
     blob_dedicated_threshold_from_metadata, blob_inline_threshold_from_metadata,
     blob_pack_file_threshold_from_metadata, preprocess_blob_batches,
 };
+use crate::index::DatasetIndexExt;
+use crate::index::scalar::{IndexDetails, fetch_index_details};
 use crate::session::Session;
 
 use super::DATA_DIR;
@@ -604,6 +603,7 @@ pub async fn do_write_fragments(
     params: WriteParams,
     storage_version: LanceFileVersion,
     target_bases_info: Option<Vec<TargetBaseInfo>>,
+    mut seed_writers: Vec<Box<dyn lance_index::scalar::seed::IndexSeedWriter>>,
 ) -> Result<Vec<Fragment>> {
     let adapter = SchemaAdapter::new(data.schema());
     let data = adapter.to_physical_stream(data);
@@ -669,6 +669,14 @@ pub async fn do_write_fragments(
             }
 
             writer.as_mut().unwrap().write(&batch_chunk).await?;
+            for seed_writer in seed_writers.iter_mut() {
+                let col_name = seed_writer.column_name().to_owned();
+                for batch in &batch_chunk {
+                    if let Some(col) = batch.column_by_name(&col_name) {
+                        seed_writer.observe_batch(col)?;
+                    }
+                }
+            }
             for batch in &batch_chunk {
                 num_rows_in_current_file += batch.num_rows() as u32;
             }
@@ -685,7 +693,9 @@ pub async fn do_write_fragments(
             if num_rows_in_current_file >= params.max_rows_per_file as u32
                 || writer.as_mut().unwrap().tell().await? >= params.max_bytes_per_file as u64
             {
-                let (num_rows, data_file) = writer.take().unwrap().finish().await?;
+                let mut w = writer.take().unwrap();
+                flush_seed_writers(w.as_mut(), &mut seed_writers).await?;
+                let (num_rows, data_file) = w.finish().await?;
                 info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path);
                 debug_assert_eq!(num_rows, num_rows_in_current_file);
                 bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get());
@@ -727,6 +737,17 @@ pub async fn do_write_fragments(
 
     // Complete the final writer
     if let Some(mut writer) = writer.take() {
+        if let Err(e) = flush_seed_writers(writer.as_mut(), &mut seed_writers).await {
+            drop(writer);
+            cleanup_data_fragments(
+                &object_store,
+                base_dir,
+                cleanup_bases.as_deref(),
+                &fragments,
+            )
+            .await;
+            return Err(e);
+        }
         match writer.finish().await {
             Ok((num_rows, data_file)) => {
                 info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path);
@@ -761,6 +782,23 @@ pub async fn do_write_fragments(
     Ok(fragments)
 }
 
+/// Flush all seed writers into the given file writer, embedding seed buffers
+/// and schema metadata before `finish()` is called.
+async fn flush_seed_writers(
+    writer: &mut dyn GenericWriter,
+    seed_writers: &mut [Box<dyn lance_index::scalar::seed::IndexSeedWriter>],
+) -> Result<()> {
+    for seed_writer in seed_writers.iter_mut() {
+        if let Some(bytes) = seed_writer.finish()? {
+            let buf_index = writer.add_global_buffer(bytes).await?;
+            let key = seed_writer.schema_metadata_key();
+            let value = seed_writer.schema_metadata_value(buf_index);
+            writer.add_schema_metadata(key, value);
+        }
+    }
+    Ok(())
+}
+
 /// Best-effort cleanup of data files for fragments that were written but not committed.
 ///
 /// Contract:
@@ -1328,6 +1366,8 @@ pub async fn write_fragments_internal(
         )));
     }
 
+    let seed_writers = create_seed_writers(dataset, &params, storage_version).await?;
+
     let fragments = do_write_fragments(
         dataset,
         object_store,
@@ -1337,12 +1377,62 @@ pub async fn write_fragments_internal(
         params,
         storage_version,
         target_bases_info,
+        seed_writers,
     )
     .await?;
 
     Ok((fragments, schema))
 }
 
+async fn create_seed_writers(
+    dataset: Option<&Dataset>,
+    params: &WriteParams,
+    storage_version: LanceFileVersion,
+) -> Result<Vec<Box<dyn lance_index::scalar::seed::IndexSeedWriter>>> {
+    // Seeds only make sense when appending to an existing dataset with V2 files.
+    if storage_version == LanceFileVersion::Legacy {
+        return Ok(Vec::new());
+    }
+    if !matches!(params.mode, WriteMode::Append) {
+        return Ok(Vec::new());
+    }
+    let Some(dataset) = dataset else {
+        return Ok(Vec::new());
+    };
+
+    let indices: Arc<Vec<IndexMetadata>> = dataset.load_indices().await?;
+    let mut writers: Vec<Box<dyn lance_index::scalar::seed::IndexSeedWriter>> = Vec::new();
+
+    for index in indices.iter() {
+        if index.fields.len() != 1 {
+            continue;
+        }
+        let field_id = index.fields[0];
+        let Ok(field_path) = dataset.schema().field_path(field_id) else {
+            continue;
+        };
+        let Some(data_type) = dataset.schema().field(&field_path).map(|f| f.data_type()) else {
+            continue;
+        };
+
+        let Ok(index_details) = fetch_index_details(dataset, &field_path, index).await else {
+            continue;
+        };
+        let details = IndexDetails(index_details.clone());
+        let Ok(plugin) = details.get_plugin() else {
+            continue;
+        };
+        if let Some(writer) = plugin
+            .create_seed_writer(&field_path, &data_type, &index_details)
+            .await?
+        {
+            writers.push(writer);
+        }
+    }
+
+    Ok(writers)
+}
+
 fn legacy_blob_field_path(schema: &Schema) -> Option<String> {
     schema
         .fields_pre_order()
@@ -1367,6 +1457,16 @@ pub trait GenericWriter: Send {
     async fn tell(&mut self) -> Result<u64>;
     /// Finish writing the file (flush the remaining data and write footer)
     async fn finish(&mut self) -> Result<(u32, DataFile)>;
+
+    /// Add a global buffer to the current file. Returns the 1-based buffer index.
+    /// Must be called before `finish`. No-op on legacy (V1) files (returns `Ok(1)`).
+    async fn add_global_buffer(&mut self, _buffer: Bytes) -> Result<u32> {
+        Ok(1)
+    }
+
+    /// Add a key-value pair to the file's schema metadata.
+    /// Must be called before `finish`. No-op on legacy (V1) files.
+    fn add_schema_metadata(&mut self, _key: String, _value: String) {}
 }
 
 struct V1WriterAdapter<M>
@@ -1463,6 +1563,14 @@ impl GenericWriter for V2WriterAdapter {
         );
         Ok((write_summary.num_rows as u32, data_file))
     }
+
+    async fn add_global_buffer(&mut self, buffer: Bytes) -> Result<u32> {
+        self.writer.add_global_buffer(buffer).await
+    }
+
+    fn add_schema_metadata(&mut self, key: String, value: String) {
+        self.writer.add_schema_metadata(key, value);
+    }
 }
 
 pub async fn open_writer(
@@ -1771,108 +1879,6 @@ async fn resolve_commit_handler(
     }
 }
 
-/// Create an iterator of record batch streams from the given source.
-///
-/// If `enable_retries` is true, then the source will be saved either in memory
-/// or spilled to disk to allow replaying the source in case of a failure. The
-/// source will be kept in memory if either (1) the size hint shows that
-/// there is only one batch or (2) the stream contains less than 100MB of
-/// data. Otherwise, the source will be spilled to a temporary file on disk.
-///
-/// This is used to support retries on write operations.
-async fn new_source_iter(
-    source: SendableRecordBatchStream,
-    enable_retries: bool,
-) -> Result<Box<dyn Iterator<Item = SendableRecordBatchStream> + Send + 'static>> {
-    if enable_retries {
-        let schema = source.schema();
-
-        // If size hint shows there is only one batch, spilling has no benefit, just keep that
-        // in memory. (This is a pretty common case.)
-        let size_hint = source.size_hint();
-        if size_hint.0 == 1 && size_hint.1 == Some(1) {
-            let batches: Vec<RecordBatch> = source.try_collect().await?;
-            Ok(Box::new(std::iter::repeat_with(move || {
-                Box::pin(RecordBatchStreamAdapter::new(
-                    schema.clone(),
-                    futures::stream::iter(batches.clone().into_iter().map(Ok)),
-                )) as SendableRecordBatchStream
-            })))
-        } else {
-            // Allow buffering up to 100MB in memory before spilling to disk.
-            Ok(Box::new(
-                SpillStreamIter::try_new(source, 100 * 1024 * 1024).await?,
-            ))
-        }
-    } else {
-        Ok(Box::new(std::iter::once(source)))
-    }
-}
-
-struct SpillStreamIter {
-    receiver: SpillReceiver,
-    _sender_handle: tokio::task::JoinHandle<SpillSender>,
-    // This temp dir is used to store the spilled data. It is kept alive by
-    // this struct. When this struct is dropped, the Drop implementation of
-    // tempfile::TempDir will delete the temp dir.
-    _tmp_dir: TempDir,
-}
-
-impl SpillStreamIter {
-    pub async fn try_new(
-        mut source: SendableRecordBatchStream,
-        memory_limit: usize,
-    ) -> Result<Self> {
-        let tmp_dir = tokio::task::spawn_blocking(|| {
-            TempDir::try_new()
-                .map_err(|e| Error::invalid_input(format!("Failed to create temp dir: {}", e)))
-        })
-        .await
-        .ok()
-        .expect_ok()??;
-
-        let tmp_path = tmp_dir.std_path().join("spill.arrows");
-        let (mut sender, receiver) = create_replay_spill(tmp_path, source.schema(), memory_limit);
-
-        let sender_handle = tokio::task::spawn(async move {
-            while let Some(res) = source.next().await {
-                match res {
-                    Ok(batch) => match sender.write(batch).await {
-                        Ok(_) => {}
-                        Err(e) => {
-                            sender.send_error(e);
-                            break;
-                        }
-                    },
-                    Err(e) => {
-                        sender.send_error(e);
-                        break;
-                    }
-                }
-            }
-
-            if let Err(err) = sender.finish().await {
-                sender.send_error(err);
-            }
-            sender
-        });
-
-        Ok(Self {
-            receiver,
-            _tmp_dir: tmp_dir,
-            _sender_handle: sender_handle,
-        })
-    }
-}
-
-impl Iterator for SpillStreamIter {
-    type Item = SendableRecordBatchStream;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        Some(self.receiver.read())
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -3733,6 +3739,7 @@ mod tests {
     async fn test_write_interruption_recovery() {
         use super::commit::CommitBuilder;
         use arrow_array::record_batch;
+        use lance_core::utils::tempfile::TempDir;
 
         // Create a temporary directory for testing
         let temp_dir = TempDir::default();
@@ -3880,6 +3887,7 @@ mod tests {
             WriteParams::default(),
             LanceFileVersion::V2_1,
             None,
+            Vec::new(),
         )
         .await;
 
@@ -3940,6 +3948,7 @@ mod tests {
             },
             LanceFileVersion::V2_1,
             None,
+            Vec::new(),
         )
         .await;
 
@@ -3980,6 +3989,7 @@ mod tests {
         let fragments = vec![Fragment {
             id: 0,
             files: vec![external_file, local_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(0),
@@ -4060,6 +4070,7 @@ mod tests {
         let fragments = vec![Fragment {
             id: 0,
             files: vec![base1_file, base2_file, unknown_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(0),
@@ -4164,6 +4175,7 @@ mod tests {
             },
             LanceFileVersion::V2_1,
             Some(target_bases),
+            vec![],
         )
         .await;
 
@@ -4479,4 +4491,116 @@ mod tests {
             .collect();
         assert_eq!(file_bases, vec![None, Some(1), Some(2)]);
     }
+
+    #[tokio::test]
+    async fn test_zone_map_seeds_used_during_update() {
+        use crate::Dataset;
+        use crate::index::DatasetIndexExt;
+        use crate::index::scalar::open_scalar_index;
+        use arrow::datatypes::Int32Type;
+        use lance_datagen::{BatchCount, RowCount};
+        use lance_datagen::{array, gen_batch};
+        use lance_file::reader::FileReaderOptions;
+        use lance_index::metrics::NoOpMetricsCollector;
+        use lance_index::scalar::seed::SEED_META_KEY_PREFIX;
+        use lance_index::{IndexType, scalar::ScalarIndexParams};
+        use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+        use lance_io::utils::CachedFileSize;
+
+        let tmpdir = lance_core::utils::tempfile::TempStrDir::default();
+        let uri = tmpdir.as_str();
+
+        // Step 1: Create initial dataset
+        let reader = gen_batch()
+            .col("val", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(100), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, uri, None).await.unwrap();
+
+        // Step 2: Create a zone map index with seeds explicitly enabled (Int32 defaults to off).
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::ZoneMap)
+            .with_params(&serde_json::json!({"use_seeds": true}));
+        dataset
+            .create_index(&["val"], IndexType::ZoneMap, None, &params, false)
+            .await
+            .unwrap();
+        // Step 3: Append new data - seeds should be written automatically
+        let reader = gen_batch()
+            .col("val", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(50), BatchCount::from(1));
+        let dataset = Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                data_storage_version: Some(lance_file::version::LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Step 4: Verify that the newly appended fragment has a seed embedded
+        let fragments = dataset.fragments();
+        let new_fragment = fragments.last().unwrap();
+        let data_file = new_fragment.files.first().unwrap();
+
+        let scheduler = ScanScheduler::new(
+            dataset.object_store.clone(),
+            SchedulerConfig::max_bandwidth(&dataset.object_store),
+        );
+        let path = dataset
+            .base
+            .clone()
+            .join(super::DATA_DIR)
+            .join(data_file.path.as_str());
+        let file_scheduler = scheduler
+            .open_file(&path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = lance_file::reader::FileReader::try_open(
+            file_scheduler,
+            None,
+            Default::default(),
+            &dataset.metadata_cache.file_metadata_cache(&path),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        let meta_key = format!("{}val", SEED_META_KEY_PREFIX);
+        let has_seed = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .contains_key(&meta_key);
+        assert!(
+            has_seed,
+            "Newly appended fragment should have a zone map seed in metadata"
+        );
+
+        // Step 5: Optimize the index (should use seeds)
+        let mut dataset = Dataset::open(uri).await.unwrap();
+        dataset.optimize_indices(&Default::default()).await.unwrap();
+
+        // Step 6: Query the updated index to verify it's correct
+        let dataset = Dataset::open(uri).await.unwrap();
+        let indices = dataset.load_indices().await.unwrap();
+        assert!(
+            !indices.is_empty(),
+            "Dataset should still have an index after optimization"
+        );
+
+        // Verify the index is a ZoneMap and covers all fragments
+        let index = indices.iter().find(|i| i.name.contains("val")).unwrap();
+        let scalar_index = open_scalar_index(&dataset, "val", index, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert_eq!(
+            scalar_index.index_type(),
+            IndexType::ZoneMap,
+            "Index should still be a ZoneMap after optimization"
+        );
+        let frags = scalar_index.calculate_included_frags().await.unwrap();
+        assert_eq!(frags.len(), 2, "Index should cover both fragments");
+    }
 }
diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs
index d76c2049873..7f2a0c4ca6d 100644
--- a/rust/lance/src/dataset/write/commit.rs
+++ b/rust/lance/src/dataset/write/commit.rs
@@ -43,6 +43,7 @@ pub struct CommitBuilder<'a> {
     commit_handler: Option<Arc<dyn CommitHandler>>,
     store_params: Option<ObjectStoreParams>,
     object_store: Option<Arc<ObjectStore>>,
+    source_store: Option<Arc<ObjectStore>>,
     session: Option<Arc<Session>>,
     detached: bool,
     commit_config: CommitConfig,
@@ -64,6 +65,7 @@ impl<'a> CommitBuilder<'a> {
             commit_handler: None,
             store_params: None,
             object_store: None,
+            source_store: None,
             session: None,
             detached: false,
             commit_config: Default::default(),
@@ -103,6 +105,18 @@ impl<'a> CommitBuilder<'a> {
         self
     }
 
+    /// Pass the object store of the dataset being cloned from.
+    ///
+    /// Only used by `Operation::Clone`: the source manifest is read through this store
+    /// while the new dataset is written through the destination store. This lets a clone
+    /// cross object stores/accounts (e.g. between two Azure accounts), where the source
+    /// is not reachable with the destination's credentials. Defaults to the destination
+    /// store when not set, preserving same-store behavior.
+    pub fn with_source_store(mut self, source_store: Arc<ObjectStore>) -> Self {
+        self.source_store = Some(source_store);
+        self
+    }
+
     /// Pass a commit handler to use for the dataset.
     ///
     /// Takes precedence over the destination dataset's own handler. If not
@@ -241,6 +255,9 @@ impl<'a> CommitBuilder<'a> {
             .or_else(|| self.dest.dataset().map(|ds| ds.session.clone()))
             .unwrap_or_default();
 
+        // Store used to read the source manifest for a clone (see with_source_store).
+        let source_store = self.source_store.clone();
+
         let (object_store, base_path, commit_handler) = match &self.dest {
             WriteDestination::Dataset(dataset) => (
                 dataset.object_store.clone(),
@@ -404,6 +421,7 @@ impl<'a> CommitBuilder<'a> {
         } else {
             commit_new_dataset(
                 object_store.as_ref(),
+                source_store.as_deref(),
                 commit_handler.as_ref(),
                 &base_path,
                 &transaction,
@@ -549,6 +567,7 @@ mod tests {
                 file_size_bytes: CachedFileSize::new(100),
                 base_id: None,
             }],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(10),
@@ -629,8 +648,11 @@ mod tests {
         assert_eq!(new_ds.manifest().version, 7);
         // Session should still be re-used
         // However, the dataset needs to be loaded and the read version checked out.
+        // The read version's manifest body is served from the session cache (it
+        // was cached when v1 was first created), so the checkout only pays the
+        // version-resolution head, not a manifest read.
         let io_stats = dataset.object_store.as_ref().io_stats_incremental();
-        assert_io_eq!(io_stats, read_iops, 4, "load dataset + check version");
+        assert_io_eq!(io_stats, read_iops, 3, "load dataset + check version");
         assert_io_eq!(io_stats, write_iops, 2, "write txn + manifest");
 
         // Commit transaction with URI and new session. Re-use the store
@@ -963,7 +985,7 @@ mod tests {
                 new_fragments: vec![],
                 removed_fragment_ids: vec![],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs
index 6e1db342f9c..144cc2bac6e 100644
--- a/rust/lance/src/dataset/write/insert.rs
+++ b/rust/lance/src/dataset/write/insert.rs
@@ -8,8 +8,8 @@ use arrow_array::{RecordBatch, RecordBatchIterator};
 use datafusion::execution::SendableRecordBatchStream;
 use humantime::format_duration;
 use lance_core::datatypes::{NullabilityComparison, Schema, SchemaCompareOptions};
+use lance_core::is_system_column;
 use lance_core::utils::tracing::{DATASET_WRITING_EVENT, TRACE_DATASET_EVENTS};
-use lance_core::{ROW_ADDR, ROW_ID, ROW_OFFSET};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
 use lance_io::object_store::ObjectStore;
@@ -328,9 +328,12 @@ impl<'a> InsertBuilder<'a> {
             normalized_data_schema.check_compatible(dataset.schema(), &schema_cmp_opts)?;
         }
 
-        // Make sure we aren't using any reserved column names
+        // The system columns (`_rowid`, `_rowaddr`, `_rowoffset`, and the row-version
+        // columns) are virtual: they're injected into scan results at read time and
+        // never stored. A stored column sharing one of these names would collide with
+        // the system column on read, so reject it at write time.
         for field in data_schema.fields.iter() {
-            if field.name == ROW_ID || field.name == ROW_ADDR || field.name == ROW_OFFSET {
+            if is_system_column(&field.name) {
                 return Err(Error::invalid_input_source(
                     format!(
                         "The column {} is a reserved name and cannot be used in a Lance dataset",
@@ -506,6 +509,38 @@ mod test {
         );
     }
 
+    #[rstest::rstest]
+    #[case::row_id("_rowid")]
+    #[case::row_addr("_rowaddr")]
+    #[case::row_offset("_rowoffset")]
+    #[case::row_created_at_version("_row_created_at_version")]
+    #[case::row_last_updated_at_version("_row_last_updated_at_version")]
+    #[tokio::test]
+    async fn rejects_reserved_system_column_names(#[case] reserved_name: &str) {
+        // Every system column name must be rejected on write. The row-version
+        // columns (`_row_created_at_version`, `_row_last_updated_at_version`) are
+        // computed at read time and appended by `Projection::to_schema`; a user
+        // data column sharing one of those names would otherwise pass ingest and
+        // later collide with the appended field.
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            reserved_name,
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))])
+            .unwrap();
+
+        let result = InsertBuilder::new("memory://")
+            .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()))
+            .await;
+
+        let err = result.expect_err("writing a reserved system column name should fail");
+        assert!(
+            err.to_string().contains("reserved name"),
+            "unexpected error for {reserved_name}: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn allow_overwrite_to_v2_2_without_blob_upgrade() {
         let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
diff --git a/rust/lance/src/dataset/write/merge_insert.rs b/rust/lance/src/dataset/write/merge_insert.rs
index a356b8c1d26..a325d5de037 100644
--- a/rust/lance/src/dataset/write/merge_insert.rs
+++ b/rust/lance/src/dataset/write/merge_insert.rs
@@ -61,7 +61,9 @@ use crate::{
     },
     index::DatasetIndexInternalExt,
     io::exec::{
-        AddRowAddrExec, Planner, TakeExec, project,
+        AddRowAddrExec, Planner, TakeExec,
+        filtered_read::{FilteredReadExec, FilteredReadOptions},
+        project,
         scalar_index::{IndexLookup, MapIndexExec},
         utils::ReplayExec,
     },
@@ -76,6 +78,8 @@ use datafusion::common::NullEquality;
 use datafusion::common::tree_node::{Transformed, TreeNode};
 use datafusion::error::DataFusionError;
 use datafusion::{
+    catalog::{TableProvider, streaming::StreamingTable},
+    datasource::MemTable,
     execution::{
         context::{SessionConfig, SessionContext},
         memory_pool::MemoryConsumer,
@@ -114,14 +118,15 @@ use lance_datafusion::{
     chunker::chunk_stream,
     dataframe::BatchStreamGrouper,
     exec::{
-        HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, analyze_plan, execute_plan,
-        get_session_context,
+        HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, OneShotPartitionStream,
+        analyze_plan, execute_plan, get_session_context, provider_to_stream,
     },
+    spill::spilling_table_provider,
     utils::{StreamingWriteSource, reader_to_stream},
 };
 use lance_file::version::LanceFileVersion;
 use lance_index::IndexCriteria;
-use lance_index::mem_wal::MergedGeneration;
+use lance_index::mem_wal::CompactedSsTable;
 use lance_select::RowAddrTreeMap;
 use lance_table::format::{Fragment, IndexMetadata, RowIdMeta};
 use log::info;
@@ -340,9 +345,15 @@ struct MergeInsertParams {
     // Controls whether data that is not matched by the source is deleted or not
     delete_not_matched_by_source: WhenNotMatchedBySource,
     conflict_retries: u32,
+    // When the source is a one-shot stream and `conflict_retries > 0`, the source
+    // is spilled (memory, then disk) so it can be replayed on each retry. Set to
+    // false to fail fast on contention instead of buffering the stream. Has no
+    // effect on re-scannable sources (materialized batches, files), which never
+    // spill.
+    spill_for_retry: bool,
     retry_timeout: Duration,
-    // List of MemWAL region generations to mark as merged when this commit succeeds.
-    merged_generations: Vec<MergedGeneration>,
+    // MemWAL SSTables to mark as compacted when this commit succeeds.
+    compacted_sstables: Vec<CompactedSsTable>,
     // If true, skip auto cleanup during commits. This should be set to true
     // for high frequency writes to improve performance. This is also useful
     // if the writer does not have delete permissions and the clean up would
@@ -477,8 +488,9 @@ impl MergeInsertBuilder {
                 insert_not_matched: true,
                 delete_not_matched_by_source: WhenNotMatchedBySource::Keep,
                 conflict_retries: 10,
+                spill_for_retry: true,
                 retry_timeout: Duration::from_secs(30),
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 skip_auto_cleanup: false,
                 use_index: true,
                 source_dedupe_behavior: SourceDedupeBehavior::Fail,
@@ -527,6 +539,27 @@ impl MergeInsertBuilder {
         self
     }
 
+    /// Controls whether a one-shot stream source is spilled so it can be replayed
+    /// across retries.
+    ///
+    /// When the source is a one-shot stream (e.g. [`MergeInsertJob::execute`]) and
+    /// `conflict_retries > 0`, the source is buffered in memory and spilled to disk
+    /// so each retry can re-read it. Set this to `false` to skip that buffering and
+    /// fail fast with a contention error instead of writing the stream to disk.
+    ///
+    /// This has no effect on re-scannable sources (materialized batches via
+    /// [`MergeInsertJob::execute_batches`], or a [`TableProvider`] via
+    /// [`MergeInsertJob::execute_provider`]), which are replayed directly and never
+    /// spill.
+    ///
+    /// Default is true.
+    ///
+    /// [`TableProvider`]: datafusion::catalog::TableProvider
+    pub fn spill_for_retry(&mut self, spill: bool) -> &mut Self {
+        self.params.spill_for_retry = spill;
+        self
+    }
+
     /// Set the timeout used to limit retries.
     ///
     /// This is the maximum time to spend on the operation before giving up. At
@@ -569,10 +602,12 @@ impl MergeInsertBuilder {
         self
     }
 
-    /// Mark MemWAL region generations as merged when this commit succeeds.
-    /// This updates the merged_generations in the MemWAL Index atomically with the data commit.
-    pub fn mark_generations_as_merged(&mut self, generations: Vec<MergedGeneration>) -> &mut Self {
-        self.params.merged_generations.extend(generations);
+    /// Mark MemWAL SSTables as compacted when this commit succeeds.
+    ///
+    /// This updates `compacted_sstables` in the MemWAL index atomically with
+    /// the data commit.
+    pub fn mark_sstables_as_compacted(&mut self, sstables: Vec<CompactedSsTable>) -> &mut Self {
+        self.params.compacted_sstables.extend(sstables);
         self
     }
 
@@ -698,6 +733,16 @@ enum SchemaComparison {
     Subschema,
 }
 
+/// Wrap a one-shot stream in a non-replayable [`StreamingTable`] provider.
+///
+/// The provider can only be scanned once (its single partition hands out the
+/// underlying stream), so it must not be used where retries may re-scan it.
+fn one_shot_provider(stream: SendableRecordBatchStream) -> Result<Arc<dyn TableProvider>> {
+    let schema = stream.schema();
+    let partition = Arc::new(OneShotPartitionStream::new(stream));
+    Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?))
+}
+
 impl MergeInsertJob {
     pub async fn execute_reader(
         self,
@@ -831,24 +876,35 @@ impl MergeInsertJob {
             index_mapper_input,
         ));
 
-        // If requested, add row addresses to the output
-        if add_row_addr {
-            let pos = index_mapper.schema().fields().len(); // Add to end
-            index_mapper = Arc::new(AddRowAddrExec::try_new(
-                index_mapper,
-                self.dataset.clone(),
-                pos,
-            )?);
-        }
-
-        // 4 - Take the mapped row ids
+        // 4 - Take the mapped row ids (TakeExec stays for legacy storage:
+        //     the v1 reader cannot serve a FilteredReadExec)
         let projection = self
             .dataset
             .empty_projection()
             .union_arrow_schema(schema.as_ref(), OnMissing::Error)?;
-        let mut target =
+        let mut target: Arc<dyn ExecutionPlan> = if self.dataset.is_legacy_storage() {
+            if add_row_addr {
+                let pos = index_mapper.schema().fields().len(); // Add to end
+                index_mapper = Arc::new(AddRowAddrExec::try_new(
+                    index_mapper,
+                    self.dataset.clone(),
+                    pos,
+                )?);
+            }
             Arc::new(TakeExec::try_new(self.dataset.clone(), index_mapper, projection)?.unwrap())
-                as Arc<dyn ExecutionPlan>;
+        } else {
+            // Keep the mapped row ids; the read synthesizes the row addresses
+            // if requested (no AddRowAddrExec needed)
+            let mut projection = projection.with_row_id();
+            if add_row_addr {
+                projection = projection.with_row_addr();
+            }
+            Arc::new(FilteredReadExec::try_new(
+                self.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(index_mapper),
+            )?)
+        };
 
         // 5 - Take puts the row id and row addr at the beginning.  A full scan (used when there is
         //     no scalar index) puts the row id and addr at the end.  We need to match these up so
@@ -1113,7 +1169,7 @@ impl MergeInsertJob {
         // sort node so each input batch fits in the memory pool.
         let capped_plan = sorted_plan
             .transform_down(|node| {
-                if node.as_any().downcast_ref::<SortExec>().is_some() {
+                if node.downcast_ref::<SortExec>().is_some() {
                     let children = node.children();
                     let new_children: Vec<Arc<dyn ExecutionPlan>> = children
                         .into_iter()
@@ -1289,9 +1345,16 @@ impl MergeInsertJob {
                     // will be the original source data, and all subsequent batches
                     // will be updates.
                     let mut source_batches = Vec::with_capacity(batches.len() + 1);
-                    source_batches.push(batches[0].clone()); // placeholder for source data
+                    // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) so every
+                    // batch is in physical format, matching what the updater reads from the
+                    // fragment. `convert_json_columns` is a no-op clone when there is nothing
+                    // to convert, so it can be applied unconditionally. The first entry is a
+                    // placeholder for the source data (overwritten each iteration below); it
+                    // must be converted too, otherwise its schema would diverge from the rest.
+                    source_batches.push(convert_json_columns(&batches[0]).map_err(Error::from)?);
                     for batch in &batches {
-                        source_batches.push(batch.drop_column(ROW_ADDR)?);
+                        let dropped = batch.drop_column(ROW_ADDR)?;
+                        source_batches.push(convert_json_columns(&dropped).map_err(Error::from)?);
                     }
 
                     // This function is here to help rustc with lifetimes.
@@ -1590,24 +1653,136 @@ impl MergeInsertJob {
         ))
     }
 
-    /// Executes the merge insert job
+    /// Executes the merge insert job from a one-shot stream source.
     ///
     /// This will take in the source, merge it with the existing target data, and insert new
-    /// rows, update existing rows, and delete existing rows
+    /// rows, update existing rows, and delete existing rows.
+    ///
+    /// A stream can only be read once, so when `conflict_retries > 0` the stream is
+    /// spilled (in memory, then to disk) so it can be replayed on each retry. See
+    /// [`MergeInsertBuilder::spill_for_retry`] to fail fast instead, and
+    /// [`Self::execute_batches`] / [`Self::execute_provider`] for re-scannable
+    /// sources that never spill.
     pub async fn execute(
         self,
         source: SendableRecordBatchStream,
     ) -> Result<(Arc<Dataset>, MergeStats)> {
-        let source_iter = super::new_source_iter(source, self.params.conflict_retries > 0).await?;
+        let (provider, replayable) = self.stream_source_to_provider(source).await?;
+        self.execute_inner(provider, replayable).await
+    }
+
+    /// Executes the merge insert job from a re-scannable [`TableProvider`].
+    ///
+    /// This is the canonical entry point: [`Self::execute`] and
+    /// [`Self::execute_batches`] are thin wrappers that build a provider and call
+    /// this method. Because a provider can be scanned repeatedly, retries re-read
+    /// the source directly and never spill to disk. The provider's reported
+    /// statistics (e.g. from a [`MemTable`] or file source) also let DataFusion
+    /// optimize the merge join.
+    ///
+    /// [`MemTable`]: datafusion::datasource::MemTable
+    pub async fn execute_provider(
+        self,
+        provider: Arc<dyn TableProvider>,
+    ) -> Result<(Arc<Dataset>, MergeStats)> {
+        // A genuine TableProvider is re-scannable by contract, so retries are safe.
+        self.execute_inner(provider, true).await
+    }
+
+    /// Executes the merge insert job from materialized record batches.
+    ///
+    /// The batches are wrapped in an in-memory [`MemTable`], which is re-scannable
+    /// (retries replay from memory, never spilling) and reports exact statistics to
+    /// the merge join. This is the preferred entry point when the full source is
+    /// already in memory.
+    pub async fn execute_batches(
+        self,
+        batches: Vec<RecordBatch>,
+    ) -> Result<(Arc<Dataset>, MergeStats)> {
+        let provider = self.batches_to_provider(batches)?;
+        self.execute_inner(provider, true).await
+    }
+
+    /// Like [`Self::execute_batches`] but returns the uncommitted transaction.
+    ///
+    /// Use [`CommitBuilder`] to commit the returned transaction.
+    pub async fn execute_uncommitted_batches(
+        self,
+        batches: Vec<RecordBatch>,
+    ) -> Result<UncommittedMergeInsert> {
+        let provider = self.batches_to_provider(batches)?;
+        self.execute_uncommitted_impl(provider).await
+    }
+
+    /// Wrap materialized batches in a multi-partition in-memory [`MemTable`].
+    fn batches_to_provider(&self, batches: Vec<RecordBatch>) -> Result<Arc<dyn TableProvider>> {
+        let schema = batches
+            .first()
+            .map(|batch| batch.schema())
+            .unwrap_or_else(|| Arc::new(Schema::from(self.dataset.schema())));
+        // Spread batches across partitions so the source can be scanned in parallel
+        // and reports per-partition statistics. A single inner Vec would be one
+        // partition with no parallelism.
+        let partitions = Self::batches_into_partitions(batches);
+        Ok(Arc::new(MemTable::try_new(schema, partitions)?))
+    }
+
+    /// Distribute batches round-robin across up to `num_compute_intensive_cpus`
+    /// partitions, so a [`MemTable`] built from them can be scanned in parallel.
+    /// Always returns at least one (possibly empty) partition so an empty source
+    /// still produces a valid provider.
+    fn batches_into_partitions(batches: Vec<RecordBatch>) -> Vec<Vec<RecordBatch>> {
+        let num_partitions = batches.len().min(get_num_compute_intensive_cpus()).max(1);
+        let mut partitions = vec![Vec::new(); num_partitions];
+        for (idx, batch) in batches.into_iter().enumerate() {
+            partitions[idx % num_partitions].push(batch);
+        }
+        partitions
+    }
+
+    /// Wrap a one-shot stream source in a provider, returning whether it can be
+    /// replayed across retries.
+    ///
+    /// With retries enabled and spilling allowed, the stream is drained into a
+    /// replayable spill (memory up to 100MB, then disk). Otherwise the stream is
+    /// wrapped in a non-replayable one-shot provider and any conflict fails fast.
+    async fn stream_source_to_provider(
+        &self,
+        source: SendableRecordBatchStream,
+    ) -> Result<(Arc<dyn TableProvider>, bool)> {
+        if self.params.conflict_retries > 0 && self.params.spill_for_retry {
+            // Allow buffering up to 100MB in memory before spilling to disk.
+            let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?;
+            Ok((provider, true))
+        } else {
+            Ok((one_shot_provider(source)?, false))
+        }
+    }
+
+    /// Run the retry loop against a provider, re-scanning it on each attempt.
+    ///
+    /// `replayable` indicates whether the provider can be scanned more than once.
+    /// When it cannot (a one-shot stream that was not spilled), retries are
+    /// disabled so we never scan it twice; the operation runs once and surfaces any
+    /// commit conflict directly.
+    async fn execute_inner(
+        self,
+        provider: Arc<dyn TableProvider>,
+        replayable: bool,
+    ) -> Result<(Arc<Dataset>, MergeStats)> {
         let dataset = self.dataset.clone();
         let config = RetryConfig {
-            max_retries: self.params.conflict_retries,
+            max_retries: if replayable {
+                self.params.conflict_retries
+            } else {
+                0
+            },
             retry_timeout: self.params.retry_timeout,
         };
 
-        let wrapper = MergeInsertJobWithIterator {
+        let wrapper = MergeInsertJobWithProvider {
             job: self,
-            source_iter: Arc::new(Mutex::new(source_iter)),
+            provider,
             attempt_count: Arc::new(AtomicU32::new(0)),
         };
 
@@ -1622,7 +1797,8 @@ impl MergeInsertJob {
         source: impl StreamingWriteSource,
     ) -> Result<UncommittedMergeInsert> {
         let stream = source.into_stream();
-        self.execute_uncommitted_impl(stream).await
+        self.execute_uncommitted_impl(one_shot_provider(stream)?)
+            .await
     }
 
     fn create_plan_join_type(&self) -> JoinType {
@@ -1640,18 +1816,32 @@ impl MergeInsertJob {
         }
     }
 
-    async fn create_plan(
-        self,
-        source: SendableRecordBatchStream,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
+    async fn create_plan(self, provider: Arc<dyn TableProvider>) -> Result<Arc<dyn ExecutionPlan>> {
         // Goal: we shouldn't manually have to specify which columns to scan.
         //       DataFusion's optimizer should be able to automatically perform
         //       projection pushdown for us.
         // Goal: we shouldn't have to add new branches in this code to handle
         //       indexed vs non-indexed cases. That should be handled by optimizer rules.
-        let session_config = SessionConfig::default();
-        let session_ctx = SessionContext::new_with_config(session_config);
-        let scan = session_ctx.read_lance_unordered(self.dataset.clone(), true, true)?;
+        let session_ctx = SessionContext::new();
+        let binary_blob_field_ids = self
+            .dataset
+            .schema()
+            .fields_pre_order()
+            .filter(|field| field.is_blob() && !field.is_blob_v2())
+            .map(|field| field.id as u32)
+            .collect();
+        let target_provider = Arc::new(
+            crate::datafusion::dataframe::LanceTableProvider::new_with_ordering(
+                self.dataset.clone(),
+                true,
+                true,
+                false,
+            )
+            .with_blob_handling(lance_core::datatypes::BlobHandling::SomeBlobsBinary(
+                binary_blob_field_ids,
+            )),
+        );
+        let scan = session_ctx.read_table(target_provider)?;
         // Wrap column names in double quotes to preserve case (DataFusion lowercases unquoted identifiers)
         let on_cols = self
             .params
@@ -1660,7 +1850,11 @@ impl MergeInsertJob {
             .map(|name| format!("\"{}\"", name))
             .collect::<Vec<_>>();
         let on_cols_refs = on_cols.iter().map(|s| s.as_str()).collect::<Vec<_>>();
-        let source_df = session_ctx.read_one_shot(source)?;
+        // Plan against the provider directly so its statistics reach the optimizer.
+        // The merge write node requires a single-partition input, so the optimizer
+        // coalesces a multi-partition provider for us (see
+        // `FullSchemaMergeInsertExec::required_input_distribution`).
+        let source_df = session_ctx.read_table(provider)?;
         // Capture the source field names *before* aliasing / joining so we
         // can tell which dataset columns are missing from the source and
         // need to be filled from the target side of the join below.
@@ -1739,14 +1933,14 @@ impl MergeInsertJob {
 
     async fn execute_uncommitted_v2(
         self,
-        source: SendableRecordBatchStream,
+        provider: Arc<dyn TableProvider>,
     ) -> Result<(
         Transaction,
         MergeStats,
         Option<RowAddrTreeMap>,
         Option<KeyExistenceFilter>,
     )> {
-        let plan = self.create_plan(source).await?;
+        let plan = self.create_plan(provider).await?;
 
         // Execute the plan
         // Assert that we have exactly one partition since we're designed for single-partition execution
@@ -1780,8 +1974,7 @@ impl MergeInsertJob {
 
         // Extract merge stats from the execution plan
         let (stats, transaction, affected_rows, inserted_rows_filter) = if let Some(full_exec) =
-            plan.as_any()
-                .downcast_ref::<exec::FullSchemaMergeInsertExec>()
+            plan.downcast_ref::<exec::FullSchemaMergeInsertExec>()
         {
             let stats = full_exec.merge_stats().ok_or_else(|| {
                 Error::internal("Merge stats not available - execution may not have completed")
@@ -1792,10 +1985,7 @@ impl MergeInsertJob {
             let affected_rows = full_exec.affected_rows().map(RowAddrTreeMap::from);
             let inserted_rows_filter = full_exec.inserted_rows_filter();
             (stats, transaction, affected_rows, inserted_rows_filter)
-        } else if let Some(delete_exec) = plan
-            .as_any()
-            .downcast_ref::<exec::DeleteOnlyMergeInsertExec>()
-        {
+        } else if let Some(delete_exec) = plan.downcast_ref::<exec::DeleteOnlyMergeInsertExec>() {
             let stats = delete_exec.merge_stats().ok_or_else(|| {
                 Error::internal("Merge stats not available - execution may not have completed")
             })?;
@@ -1935,14 +2125,14 @@ impl MergeInsertJob {
 
     async fn execute_uncommitted_impl(
         self,
-        source: SendableRecordBatchStream,
+        provider: Arc<dyn TableProvider>,
     ) -> Result<UncommittedMergeInsert> {
         // Check if we can use the fast path
-        let can_use_fast_path = self.can_use_create_plan(source.schema().as_ref()).await?;
+        let can_use_fast_path = self.can_use_create_plan(provider.schema().as_ref()).await?;
 
         if can_use_fast_path {
             let (transaction, stats, affected_rows, inserted_rows_filter) =
-                self.execute_uncommitted_v2(source).await?;
+                self.execute_uncommitted_v2(provider).await?;
             return Ok(UncommittedMergeInsert {
                 transaction,
                 affected_rows,
@@ -1953,6 +2143,8 @@ impl MergeInsertJob {
 
         let target_bases_info = resolve_target_bases(&self.dataset, &self.params).await?;
 
+        // The slow path consumes a single stream; adapt the provider back into one.
+        let source = provider_to_stream(provider).await?;
         let source_schema = source.schema();
         let lance_schema = lance_core::datatypes::Schema::try_from(source_schema.as_ref())?;
         let full_schema = self.dataset.schema();
@@ -2014,7 +2206,7 @@ impl MergeInsertJob {
                 } else {
                     removed_row_ids
                 };
-            let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec.into_iter());
+            let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec);
 
             let (updated_fragments, removed_fragment_ids) =
                 Self::apply_deletions(&self.dataset, &removed_row_addrs).await?;
@@ -2024,7 +2216,7 @@ impl MergeInsertJob {
                 updated_fragments,
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations: self.params.merged_generations.clone(),
+                compacted_sstables: self.params.compacted_sstables.clone(),
                 fields_for_preserving_frag_bitmap: full_schema
                     .fields
                     .iter()
@@ -2063,7 +2255,7 @@ impl MergeInsertJob {
                 updated_fragments,
                 new_fragments,
                 fields_modified,
-                merged_generations: self.params.merged_generations.clone(),
+                compacted_sstables: self.params.compacted_sstables.clone(),
                 fields_for_preserving_frag_bitmap: vec![], // in-place update do not affect preserving frag bitmap
                 update_mode: Some(RewriteColumns),
                 inserted_rows_filter: None, // not implemented for v1
@@ -2125,7 +2317,7 @@ impl MergeInsertJob {
                         removed_row_ids
                     };
 
-                Ok(RoaringTreemap::from_iter(removed_row_addr_vec.into_iter()))
+                Ok(RoaringTreemap::from_iter(removed_row_addr_vec))
             }
             .await;
             let removed_row_addrs = match post_write_result {
@@ -2165,7 +2357,7 @@ impl MergeInsertJob {
                 // On this path we only make deletions against updated_fragments and will not
                 // modify any field values.
                 fields_modified: vec![],
-                merged_generations: self.params.merged_generations.clone(),
+                compacted_sstables: self.params.compacted_sstables.clone(),
                 fields_for_preserving_frag_bitmap: full_schema
                     .fields
                     .iter()
@@ -2278,7 +2470,9 @@ impl MergeInsertJob {
 
         // Clone self since create_plan consumes the job
         let cloned_job = self.clone();
-        let plan = cloned_job.create_plan(Box::pin(stream)).await?;
+        let plan = cloned_job
+            .create_plan(one_shot_provider(Box::pin(stream))?)
+            .await?;
         let display = DisplayableExecutionPlan::new(plan.as_ref());
 
         Ok(format!("{}", display.indent(verbose)))
@@ -2311,7 +2505,7 @@ impl MergeInsertJob {
 
         // Clone self since create_plan consumes the job
         let cloned_job = self.clone();
-        let plan = cloned_job.create_plan(source).await?;
+        let plan = cloned_job.create_plan(one_shot_provider(source)?).await?;
 
         // Use the analyze_plan function from lance_datafusion, but strip out the wrapper lines
         let options = LanceExecutionOptions::default();
@@ -2361,15 +2555,15 @@ pub struct UncommittedMergeInsert {
     pub inserted_rows_filter: Option<KeyExistenceFilter>,
 }
 
-/// Wrapper struct that combines MergeInsertJob with the source iterator for retry functionality
+/// Wrapper struct that combines MergeInsertJob with the source provider for retry functionality
 #[derive(Clone)]
-struct MergeInsertJobWithIterator {
+struct MergeInsertJobWithProvider {
     job: MergeInsertJob,
-    source_iter: Arc<Mutex<Box<dyn Iterator<Item = SendableRecordBatchStream> + Send + 'static>>>,
+    provider: Arc<dyn TableProvider>,
     attempt_count: Arc<AtomicU32>,
 }
 
-impl RetryExecutor for MergeInsertJobWithIterator {
+impl RetryExecutor for MergeInsertJobWithProvider {
     type Data = UncommittedMergeInsert;
     type Result = (Arc<Dataset>, MergeStats);
 
@@ -2377,10 +2571,11 @@ impl RetryExecutor for MergeInsertJobWithIterator {
         // Increment attempt counter
         self.attempt_count.fetch_add(1, Ordering::SeqCst);
 
-        // We need to get a fresh stream for each retry attempt
-        // The source_iter provides unlimited streams from the same source data
-        let stream = self.source_iter.lock().unwrap().next().unwrap();
-        self.job.clone().execute_uncommitted_impl(stream).await
+        // Re-scan the provider on each retry attempt.
+        self.job
+            .clone()
+            .execute_uncommitted_impl(self.provider.clone())
+            .await
     }
 
     async fn commit(&self, dataset: Arc<Dataset>, mut data: Self::Data) -> Result<Self::Result> {
@@ -6524,7 +6719,10 @@ mod tests {
         let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16));
         let new_data_stream = reader_to_stream(Box::new(new_data));
 
-        let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
+        let plan = merge_insert_job
+            .create_plan(one_shot_provider(new_data_stream).unwrap())
+            .await
+            .unwrap();
 
         // Assert the plan structure using portable plan matching
         // The optimized plan should have:
@@ -6577,7 +6775,10 @@ mod tests {
         let new_data_stream = reader_to_stream(Box::new(new_data));
 
         // This should use the fast path (execute_uncommitted_v2)
-        let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
+        let plan = merge_insert_job
+            .create_plan(one_shot_provider(new_data_stream).unwrap())
+            .await
+            .unwrap();
 
         // The optimized plan should use Inner join instead of Right join since we're not
         // inserting unmatched rows.  The sentinel IS NOT NULL condition is folded away by
@@ -6625,7 +6826,10 @@ mod tests {
         let new_data_reader = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16));
         let new_data_stream = reader_to_stream(Box::new(new_data_reader));
 
-        let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
+        let plan = merge_insert_job
+            .create_plan(one_shot_provider(new_data_stream).unwrap())
+            .await
+            .unwrap();
 
         // The optimized plan should use Inner join and include the UpdateIf condition.
         // The sentinel IS NOT NULL condition is folded away (sentinel is lit(true)).
@@ -6677,7 +6881,10 @@ mod tests {
 
         // Should reach the v2 fast path (`create_plan` + FullSchemaMergeInsertExec).
         // Dropping to v1 here would return an error from create_plan instead.
-        let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap();
+        let plan = merge_insert_job
+            .create_plan(one_shot_provider(new_data_stream).unwrap())
+            .await
+            .unwrap();
 
         // The join is Right because we keep unmatched source rows (InsertAll)
         // but discard unmatched target rows (DoNothing on when_matched,
@@ -7913,7 +8120,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
         let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
         assert!(plan.contains("HashJoinExec"));
         assert!(plan.contains("join_type=Full"));
-        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.lines().any(|line| line.contains("HashJoinExec")
+                && line.contains("projection=[")
+                && line.contains("_rowid")),
+            "join should push down a projection that retains _rowid: {plan}"
+        );
         assert!(
             plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
             "target-side scan should prune the FSL payload from the join build side: {plan}"
@@ -7981,7 +8193,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
         let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
         assert!(plan.contains("HashJoinExec"));
         assert!(plan.contains("join_type=Full"));
-        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.lines().any(|line| line.contains("HashJoinExec")
+                && line.contains("projection=[")
+                && line.contains("_rowid")),
+            "join should push down a projection that retains _rowid: {plan}"
+        );
         assert!(
             plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
             "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}"
@@ -9602,7 +9819,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             [Ok(new_batch.clone())],
             schema.clone(),
         )));
-        let plan = plan_job.create_plan(plan_stream).await.unwrap();
+        let plan = plan_job
+            .create_plan(one_shot_provider(plan_stream).unwrap())
+            .await
+            .unwrap();
         assert_plan_node_equals(
             plan,
             "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing
@@ -9684,7 +9904,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             [Ok(new_batch.clone())],
             id_only_schema.clone(),
         )));
-        let plan = plan_job.create_plan(plan_stream).await.unwrap();
+        let plan = plan_job
+            .create_plan(one_shot_provider(plan_stream).unwrap())
+            .await
+            .unwrap();
         assert_plan_node_equals(
             plan,
             "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing
@@ -9766,7 +9989,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             [Ok(new_batch.clone())],
             schema.clone(),
         )));
-        let plan = plan_job.create_plan(plan_stream).await.unwrap();
+        let plan = plan_job
+            .create_plan(one_shot_provider(plan_stream).unwrap())
+            .await
+            .unwrap();
         assert_plan_node_equals(
             plan,
             "MergeInsert: on=[key], when_matched=Delete, when_not_matched=InsertAll, when_not_matched_by_source=Keep...THEN 2 WHEN...THEN 3 ELSE 0 END as __action]...projection=[key, value, filterme]"
@@ -9871,7 +10097,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             [Ok(non_matching_batch.clone())],
             schema.clone(),
         )));
-        let plan = plan_job.create_plan(plan_stream).await.unwrap();
+        let plan = plan_job
+            .create_plan(one_shot_provider(plan_stream).unwrap())
+            .await
+            .unwrap();
         assert_plan_node_equals(
             plan,
             "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing
@@ -9991,7 +10220,10 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
                 [Ok(new_batch)],
                 schema.clone(),
             )));
-            let plan = job.create_plan(plan_stream).await.unwrap();
+            let plan = job
+                .create_plan(one_shot_provider(plan_stream).unwrap())
+                .await
+                .unwrap();
 
             let plan_str = datafusion::physical_plan::displayable(plan.as_ref())
                 .indent(true)
@@ -10082,7 +10314,7 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
         impl std::error::Error for MyTestError {}
 
         #[tokio::test]
-        async fn test_merge_insert_execute_reader_preserves_external_error() {
+        async fn test_merge_insert_execute_reader_preserves_error_message() {
             let schema = Arc::new(ArrowSchema::new(vec![
                 ArrowField::new("key", DataType::Int32, false),
                 ArrowField::new("value", DataType::Int32, false),
@@ -10119,14 +10351,14 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
                 .execute_reader(Box::new(reader) as Box<dyn RecordBatchReader + Send>)
                 .await;
 
-            match result {
-                Err(Error::External { source }) => {
-                    let original = source.downcast_ref::<MyTestError>().unwrap();
-                    assert_eq!(original.code, error_code);
-                }
-                Err(other) => panic!("Expected External, got: {:?}", other),
-                Ok(_) => panic!("Expected error"),
-            }
+            // The source error is routed through the merge plan, which shares it
+            // across join partitions, so its concrete type is not recoverable. The
+            // message must still reach the caller.
+            let err = result.expect_err("expected the source error to surface");
+            assert!(
+                err.to_string().contains("merge insert failure"),
+                "source error message should be preserved; got: {err}"
+            );
         }
     }
 
@@ -11311,4 +11543,580 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             take_meta_field
         );
     }
+
+    #[tokio::test]
+    async fn test_merge_insert_subschema_with_json_columns() {
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::ARROW_JSON_EXT_NAME;
+
+        // Create a dataset with an Arrow JSON extension column
+        let test_dir = TempStrDir::default();
+        let mut json_metadata = HashMap::new();
+        json_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            ARROW_JSON_EXT_NAME.to_string(),
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("name", DataType::Utf8, true),
+            Field::new("score", DataType::Int64, true),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])),
+                Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),
+                Arc::new(Int64Array::from(vec![10, 20, 30, 40, 50])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"x":1}"#,
+                    r#"{"x":2}"#,
+                    r#"{"x":3}"#,
+                    r#"{"x":4}"#,
+                    r#"{"x":5}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let dataset = Arc::new(
+            Dataset::write(reader, test_dir.as_ref(), None)
+                .await
+                .unwrap(),
+        );
+
+        // Perform a subschema merge_insert: only update "meta" column (JSON type)
+        // This exercises the update_fragments path with interleave_batches
+        let update_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata),
+        ]));
+        let update_batch = RecordBatch::try_new(
+            update_schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![2, 4])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true,"id":2}"#,
+                    r#"{"updated":true,"id":4}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let update_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(update_batch)],
+            update_schema,
+        ));
+        let stream = reader_to_stream(update_reader);
+
+        // Execute merge_insert with subschema (only id + meta columns)
+        let mut builder =
+            MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap();
+        builder.when_matched(WhenMatched::UpdateAll);
+        builder.when_not_matched(WhenNotMatched::DoNothing);
+        let job = builder.try_build().unwrap();
+        let (updated_dataset, stats) = job.execute(stream).await.unwrap();
+
+        // Verify: the merge should not fail with type mismatch
+        assert_eq!(stats.num_updated_rows, 2);
+
+        // Read back and verify the JSON column was updated correctly
+        let batches = updated_dataset
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let result = concat_batches(&batches[0].schema(), &batches).unwrap();
+        assert_eq!(result.num_rows(), 5);
+
+        // Verify the "score" column (not in update) is preserved, and "meta" updated
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        let scores = result
+            .column_by_name("score")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        let metas = result
+            .column_by_name("meta")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        for i in 0..5 {
+            let id = ids.value(i);
+            let score = scores.value(i);
+            let meta = metas.value(i);
+            // score = id * 10, regardless of row order
+            assert_eq!(score, id * 10, "id={} score mismatch", id);
+            if id == 2 || id == 4 {
+                assert!(
+                    meta.contains("updated"),
+                    "id={} should have updated meta, got: {}",
+                    id,
+                    meta
+                );
+            } else {
+                assert!(
+                    meta.contains("\"x\""),
+                    "id={} should have original meta, got: {}",
+                    id,
+                    meta
+                );
+            }
+        }
+    }
+
+    fn id_value_schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new("value", DataType::UInt32, false),
+        ]))
+    }
+
+    /// `execute_provider` is the canonical entry point; a `MemTable` source merges
+    /// the same way a stream does.
+    #[tokio::test]
+    async fn test_merge_insert_execute_provider() {
+        let initial =
+            record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap();
+        let dataset = Arc::new(
+            InsertBuilder::new("memory://")
+                .execute(vec![initial])
+                .await
+                .unwrap(),
+        );
+
+        // Update id=1, insert id=3.
+        let new_data = record_batch!(("id", UInt32, [1, 3]), ("value", UInt32, [10, 30])).unwrap();
+        let provider: Arc<dyn TableProvider> = Arc::new(
+            datafusion::datasource::MemTable::try_new(new_data.schema(), vec![vec![new_data]])
+                .unwrap(),
+        );
+
+        let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap()
+            .execute_provider(provider)
+            .await
+            .unwrap();
+
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+
+        let batch = merged.scan().try_into_batch().await.unwrap();
+        let ids = batch["id"].as_primitive::<UInt32Type>();
+        let values = batch["value"].as_primitive::<UInt32Type>();
+        let merged_rows: HashMap<u32, u32> = ids
+            .values()
+            .iter()
+            .zip(values.values().iter())
+            .map(|(id, value)| (*id, *value))
+            .collect();
+        assert_eq!(
+            merged_rows,
+            HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)])
+        );
+    }
+
+    /// `execute_batches` merges materialized batches; multiple batches are spread
+    /// across partitions and merged correctly.
+    #[tokio::test]
+    async fn test_merge_insert_execute_batches() {
+        let initial =
+            record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap();
+        let dataset = Arc::new(
+            InsertBuilder::new("memory://")
+                .execute(vec![initial])
+                .await
+                .unwrap(),
+        );
+
+        // Two batches: update id=1 (batch 0), insert id=3 (batch 1).
+        let batch0 = record_batch!(("id", UInt32, [1]), ("value", UInt32, [10])).unwrap();
+        let batch1 = record_batch!(("id", UInt32, [3]), ("value", UInt32, [30])).unwrap();
+
+        let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap()
+            .execute_batches(vec![batch0, batch1])
+            .await
+            .unwrap();
+
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+
+        let batch = merged.scan().try_into_batch().await.unwrap();
+        let ids = batch["id"].as_primitive::<UInt32Type>();
+        let values = batch["value"].as_primitive::<UInt32Type>();
+        let merged_rows: HashMap<u32, u32> = ids
+            .values()
+            .iter()
+            .zip(values.values().iter())
+            .map(|(id, value)| (*id, *value))
+            .collect();
+        assert_eq!(
+            merged_rows,
+            HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)])
+        );
+    }
+
+    /// An empty batch list still produces a valid (single, empty) partition, so the
+    /// merge is a no-op and the target is unchanged.
+    #[tokio::test]
+    async fn test_merge_insert_execute_batches_empty() {
+        let initial =
+            record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap();
+        let dataset = Arc::new(
+            InsertBuilder::new("memory://")
+                .execute(vec![initial])
+                .await
+                .unwrap(),
+        );
+
+        let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap()
+            .execute_batches(vec![])
+            .await
+            .unwrap();
+
+        assert_eq!(stats.num_updated_rows, 0);
+        assert_eq!(stats.num_inserted_rows, 0);
+
+        let batch = merged.scan().try_into_batch().await.unwrap();
+        let ids = batch["id"].as_primitive::<UInt32Type>();
+        let values = batch["value"].as_primitive::<UInt32Type>();
+        let merged_rows: HashMap<u32, u32> = ids
+            .values()
+            .iter()
+            .zip(values.values().iter())
+            .map(|(id, value)| (*id, *value))
+            .collect();
+        assert_eq!(merged_rows, HashMap::from([(0, 0), (1, 0), (2, 0)]));
+    }
+
+    fn collect_exact_row_counts(plan: &Arc<dyn ExecutionPlan>, out: &mut Vec<usize>) {
+        if let Ok(stats) = plan.partition_statistics(None)
+            && let datafusion::common::stats::Precision::Exact(n) = stats.num_rows
+        {
+            out.push(n);
+        }
+        for child in plan.children() {
+            collect_exact_row_counts(child, out);
+        }
+    }
+
+    /// Use case 3: planning against the provider exposes its exact source
+    /// statistics to the optimizer.
+    #[tokio::test]
+    async fn test_merge_insert_source_statistics_in_plan() {
+        let schema = id_value_schema();
+        let target_rows = 1000u32;
+        let initial = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from_iter_values(0..target_rows)),
+                Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n(
+                    0,
+                    target_rows as usize,
+                ))),
+            ],
+        )
+        .unwrap();
+        let dataset = Arc::new(
+            InsertBuilder::new("memory://")
+                .execute(vec![initial])
+                .await
+                .unwrap(),
+        );
+
+        // A small source whose exact row count is distinct from the target's.
+        let source_rows = 10usize;
+        let new_data = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from_iter_values(0..source_rows as u32)),
+                Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n(
+                    1,
+                    source_rows,
+                ))),
+            ],
+        )
+        .unwrap();
+        let provider: Arc<dyn TableProvider> =
+            Arc::new(MemTable::try_new(schema.clone(), vec![vec![new_data]]).unwrap());
+
+        let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .try_build()
+            .unwrap();
+
+        // The provider's exact row count reaches the plan's statistics.
+        let plan = job.create_plan(provider).await.unwrap();
+        let mut row_counts = Vec::new();
+        collect_exact_row_counts(&plan, &mut row_counts);
+        assert!(
+            row_counts.contains(&source_rows),
+            "source provider's exact row count ({source_rows}) should reach the plan; got {row_counts:?}"
+        );
+    }
+
+    /// With a one-shot stream source and `spill_for_retry(false)`, a commit
+    /// conflict fails fast instead of replaying the stream. The non-replayable
+    /// one-shot provider must be scanned exactly once (scanning it twice would
+    /// panic), proving retries are disabled even though `conflict_retries > 0`.
+    #[tokio::test]
+    async fn test_merge_insert_spill_for_retry_false_fails_fast() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false).with_metadata(
+                vec![(
+                    "lance-schema:unenforced-primary-key".to_string(),
+                    "true".to_string(),
+                )]
+                .into_iter()
+                .collect(),
+            ),
+            Field::new("value", DataType::UInt32, false),
+        ]));
+        let initial = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![0, 1, 2, 3])),
+                Arc::new(UInt32Array::from(vec![0, 0, 0, 0])),
+            ],
+        )
+        .unwrap();
+        let dataset = Arc::new(
+            InsertBuilder::new("memory://")
+                .execute(vec![initial])
+                .await
+                .unwrap(),
+        );
+
+        // Merge insert job based on version 1, with retries enabled but spilling off.
+        let new_data = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![100])),
+                Arc::new(UInt32Array::from(vec![1])),
+            ],
+        )
+        .unwrap();
+        let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .conflict_retries(10)
+            .spill_for_retry(false)
+            .try_build()
+            .unwrap();
+
+        // An append commits first (version 2), so the merge built on version 1 hits
+        // an unresolvable conflict on commit.
+        let append_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![50])),
+                Arc::new(UInt32Array::from(vec![2])),
+            ],
+        )
+        .unwrap();
+        InsertBuilder::new(dataset.clone())
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute(vec![append_batch])
+            .await
+            .unwrap();
+
+        let source = RecordBatchStreamAdapter::new(
+            schema.clone(),
+            futures::stream::iter(vec![Ok(new_data)]),
+        );
+        let merge_result = job
+            .execute(Box::pin(source) as SendableRecordBatchStream)
+            .await;
+
+        assert!(
+            matches!(
+                merge_result,
+                Err(crate::Error::TooMuchWriteContention { .. })
+            ),
+            "Expected fail-fast TooMuchWriteContention, got: {:?}",
+            merge_result
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_with_blob_v1_source_provides_blob() {
+        use arrow_array::LargeBinaryArray;
+        use arrow_schema::Schema as ArrowSchema;
+        use lance_arrow::BLOB_META_KEY;
+
+        let test_dir = TempStrDir::default();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("blobs", DataType::LargeBinary, true).with_metadata(HashMap::from([(
+                BLOB_META_KEY.to_string(),
+                "true".to_string(),
+            )])),
+            Field::new("id", DataType::Int64, true),
+            Field::new("other", DataType::Int64, true),
+        ]));
+        let make_batch = |blob_values: Vec<Option<&[u8]>>, ids, others| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(LargeBinaryArray::from(blob_values)),
+                    Arc::new(Int64Array::from(ids)),
+                    Arc::new(Int64Array::from(others)),
+                ],
+            )
+            .unwrap()
+        };
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(
+                    vec![Ok(make_batch(
+                        vec![Some(b"foo"), Some(b"bar")],
+                        vec![0, 1],
+                        vec![10, 20],
+                    ))],
+                    schema.clone(),
+                ),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_1),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+        let source = Box::new(RecordBatchIterator::new(
+            vec![Ok(make_batch(
+                vec![Some(b"baz"), Some(b"qux")],
+                vec![1, 2],
+                vec![200, 300],
+            ))],
+            schema,
+        ));
+
+        let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap();
+        let (new_dataset, _) = job.execute_reader(source).await.unwrap();
+        let blobs = new_dataset
+            .take_blobs_by_indices(&[0, 1, 2], "blobs")
+            .await
+            .unwrap();
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"foo"
+        );
+        assert_eq!(
+            blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"baz"
+        );
+        assert_eq!(
+            blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"qux"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_with_blob_v2_source_provides_blob() {
+        use crate::{BlobArrayBuilder, blob_field};
+        use arrow_schema::Schema as ArrowSchema;
+
+        let test_dir = TempStrDir::default();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            blob_field("blobs", true),
+            Field::new("id", DataType::Int64, true),
+            Field::new("other", DataType::Int64, true),
+        ]));
+        let make_batch = |blob_values: &[&[u8]], ids, others| {
+            let mut blobs = BlobArrayBuilder::new(blob_values.len());
+            for value in blob_values {
+                blobs.push_bytes(value).unwrap();
+            }
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    blobs.finish().unwrap(),
+                    Arc::new(Int64Array::from(ids)),
+                    Arc::new(Int64Array::from(others)),
+                ],
+            )
+            .unwrap()
+        };
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(
+                    vec![Ok(make_batch(&[b"foo", b"bar"], vec![0, 1], vec![10, 20]))],
+                    schema.clone(),
+                ),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+        let source = Box::new(RecordBatchIterator::new(
+            vec![Ok(make_batch(
+                &[b"baz", b"qux"],
+                vec![1, 2],
+                vec![200, 300],
+            ))],
+            schema,
+        ));
+
+        let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap();
+        let (new_dataset, _) = job.execute_reader(source).await.unwrap();
+        let blobs = new_dataset
+            .take_blobs_by_indices(&[0, 1, 2], "blobs")
+            .await
+            .unwrap();
+        assert_eq!(
+            blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"foo"
+        );
+        assert_eq!(
+            blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"baz"
+        );
+        assert_eq!(
+            blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(),
+            b"qux"
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
index 07fad758902..2fbfd6fb925 100644
--- a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
+++ b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
@@ -216,10 +216,6 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
         "DeleteOnlyMergeInsertExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         Arc::new(arrow_schema::Schema::empty())
     }
@@ -283,7 +279,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
         let merge_stats_holder = self.merge_stats.clone();
         let transaction_holder = self.transaction.clone();
         let affected_rows_holder = self.affected_rows.clone();
-        let merged_generations = self.params.merged_generations.clone();
+        let compacted_sstables = self.params.compacted_sstables.clone();
         let source_dedupe_behavior = self.params.source_dedupe_behavior;
         let on_columns = self.params.on.clone();
 
@@ -308,7 +304,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
                 updated_fragments,
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations,
+                compacted_sstables,
                 fields_for_preserving_frag_bitmap: dataset
                     .schema()
                     .fields
diff --git a/rust/lance/src/dataset/write/merge_insert/exec/write.rs b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
index d5b51b3d97f..9e6db083156 100644
--- a/rust/lance/src/dataset/write/merge_insert/exec/write.rs
+++ b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
@@ -49,6 +49,48 @@ use crate::{
 
 use super::apply_deletions;
 
+fn blob_v2_user_view_schema(
+    input_schema: arrow_schema::SchemaRef,
+    dataset_schema: &lance_core::datatypes::Schema,
+) -> arrow_schema::SchemaRef {
+    let fields = input_schema
+        .fields()
+        .iter()
+        .map(|field| {
+            let Some(dataset_field) = dataset_schema
+                .field(field.name())
+                .filter(|dataset_field| dataset_field.is_blob_v2())
+            else {
+                return field.clone();
+            };
+            let is_descriptor = matches!(
+                field.data_type(),
+                arrow_schema::DataType::Struct(fields)
+                    if fields.iter().any(|child| child.name() == "kind")
+            );
+            if !is_descriptor {
+                return field.clone();
+            }
+            let logical_field = arrow_schema::Field::from(dataset_field);
+            Arc::new(
+                arrow_schema::Field::new(
+                    field.name(),
+                    lance_core::datatypes::BLOB_V2_USER_TYPE.clone(),
+                    field.is_nullable(),
+                )
+                .with_metadata(logical_field.metadata().clone()),
+            )
+        })
+        .collect::<Vec<_>>();
+    Arc::new(arrow_schema::Schema::new_with_metadata(
+        fields
+            .iter()
+            .map(|field| field.as_ref().clone())
+            .collect::<Vec<_>>(),
+        input_schema.metadata().clone(),
+    ))
+}
+
 /// Shared state for merge insert operations to simplify lock management
 struct MergeState {
     /// Row addresses that need to be deleted, due to a row update or delete action
@@ -797,10 +839,6 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
         "FullSchemaMergeInsertExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         Arc::new(arrow_schema::Schema::empty())
     }
@@ -868,6 +906,36 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
 
         // Execute the input plan to get the merge data stream
         let input_stream = self.input.execute(partition, context)?;
+        let has_blob_v2_columns = self
+            .dataset
+            .schema()
+            .fields_pre_order()
+            .any(|field| field.is_blob_v2());
+        let input_stream = if has_blob_v2_columns {
+            let input_schema = input_stream.schema();
+            let output_schema = blob_v2_user_view_schema(input_schema, self.dataset.schema());
+            let dataset = self.dataset.clone();
+            let dataset_schema = self.dataset.schema().clone();
+            let transformed = input_stream.then(move |batch_result| {
+                let dataset = dataset.clone();
+                let dataset_schema = dataset_schema.clone();
+                async move {
+                    let batch = batch_result?;
+                    crate::dataset::optimize::transform_blob_v2_batch(
+                        &dataset,
+                        &dataset_schema,
+                        batch,
+                        true,
+                    )
+                    .await
+                    .map_err(|error| DataFusionError::External(Box::new(error)))
+                }
+            });
+            Box::pin(RecordBatchStreamAdapter::new(output_schema, transformed))
+                as SendableRecordBatchStream
+        } else {
+            input_stream
+        };
 
         // Step 1: Create shared state and streaming processor for row addresses and write data
         // Get field IDs for the ON columns from the dataset schema
@@ -894,7 +962,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
         let transaction_holder = self.transaction.clone();
         let affected_rows_holder = self.affected_rows.clone();
         let inserted_rows_filter_holder = self.inserted_rows_filter.clone();
-        let merged_generations = self.params.merged_generations.clone();
+        let compacted_sstables = self.params.compacted_sstables.clone();
         let is_primary_key = self.is_primary_key;
         let updating_row_ids = {
             let state = merge_state.lock().unwrap();
@@ -994,7 +1062,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
                 updated_fragments,
                 new_fragments,
                 fields_modified: vec![], // No fields are modified in schema for upsert
-                merged_generations,
+                compacted_sstables,
                 // Use the full pre-order field list (not just top-level `fields`) so
                 // that nested leaf field ids are included. A merge_insert rewrites whole
                 // rows, so every field is potentially modified; omitting nested ids would
diff --git a/rust/lance/src/dataset/write/update.rs b/rust/lance/src/dataset/write/update.rs
index a87b3a4260b..2e6b282dedb 100644
--- a/rust/lance/src/dataset/write/update.rs
+++ b/rust/lance/src/dataset/write/update.rs
@@ -419,7 +419,7 @@ impl UpdateJob {
             // are moved(deleted and appended).
             // so we do not need to handle the frag bitmap of the index about it.
             fields_modified: vec![],
-            merged_generations: Vec::new(),
+            compacted_sstables: Vec::new(),
             fields_for_preserving_frag_bitmap,
             update_mode: Some(RewriteRows),
             inserted_rows_filter: None,
diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs
index f5afb48804c..8b871001acf 100644
--- a/rust/lance/src/index.rs
+++ b/rust/lance/src/index.rs
@@ -7,6 +7,7 @@
 use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, OnceLock};
+use std::time::Instant;
 
 use arrow_schema::DataType;
 use async_trait::async_trait;
@@ -24,8 +25,8 @@ use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::reader::FileReaderOptions;
 use lance_index::INDEX_METADATA_SCHEMA_KEY;
 pub use lance_index::IndexParams;
-use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex};
-use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex};
+use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex, FragReuseIndexHandle};
+use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexHandle};
 use lance_index::optimize::OptimizeOptions;
 use lance_index::pb::index::Implementation;
 pub use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
@@ -33,7 +34,7 @@ use lance_index::scalar::expression::{IndexInformationProvider, MultiQueryParser
 use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexPlugin};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering};
-use lance_index::scalar::{CreatedIndex, ScalarIndex};
+use lance_index::scalar::{CreatedIndex, ScalarIndex, index_files_to_table, table_files_to_index};
 use lance_index::vector::bq::builder::RabitQuantizer;
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HNSW;
@@ -60,7 +61,7 @@ use lance_table::io::manifest::read_manifest_indexes;
 use roaring::RoaringBitmap;
 use scalar::index_matches_criteria;
 use serde_json::json;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use uuid::Uuid;
 use vector::details::{
     derive_vector_index_type, infer_missing_vector_details, vector_details_as_json,
@@ -130,11 +131,109 @@ fn validate_segment_metadata(index_name: &str, segments: &[IndexMetadata]) -> Re
     Ok(())
 }
 
+fn collect_subtree_field_ids(field: &Field, field_ids: &mut HashSet<i32>) {
+    field_ids.insert(field.id);
+    for child in &field.children {
+        collect_subtree_field_ids(child, field_ids);
+    }
+}
+
+fn fragment_field_paths<'a>(
+    fragment: &'a Fragment,
+    indexed_field_ids: &HashSet<i32>,
+) -> HashMap<i32, &'a str> {
+    fragment
+        .files
+        .iter()
+        .flat_map(|file| {
+            file.fields
+                .iter()
+                .filter(|field_id| indexed_field_ids.contains(field_id))
+                .map(|field_id| (*field_id, file.path.as_str()))
+        })
+        .collect()
+}
+
+async fn prune_stale_segment_coverage(
+    dataset: &Dataset,
+    field_id: i32,
+    segments: &mut [IndexSegment],
+    prune_historically_missing: bool,
+    prune_newer_overlays: bool,
+) -> Result<()> {
+    let field = dataset.schema().field_by_id(field_id).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: field id {field_id} does not exist in the current schema"
+        ))
+    })?;
+    let mut indexed_field_ids = HashSet::new();
+    collect_subtree_field_ids(field, &mut indexed_field_ids);
+
+    let current_fragments = dataset
+        .fragments()
+        .iter()
+        .map(|fragment| (fragment.id as u32, fragment))
+        .collect::<HashMap<_, _>>();
+    let historical_versions = segments
+        .iter()
+        .map(IndexSegment::dataset_version)
+        .filter(|version| *version < dataset.manifest.version)
+        .collect::<HashSet<_>>();
+
+    for version in historical_versions {
+        let historical = dataset.checkout_version(version).await.map_err(|error| {
+            Error::invalid_input(format!(
+                "CreateIndex: cannot validate segment coverage built at dataset version {version}: {error}"
+            ))
+        })?;
+        let historical_fragments = historical
+            .fragments()
+            .iter()
+            .map(|fragment| (fragment.id as u32, fragment))
+            .collect::<HashMap<_, _>>();
+
+        for segment in segments
+            .iter_mut()
+            .filter(|segment| segment.dataset_version() == version)
+        {
+            let stale_fragments = segment
+                .fragment_bitmap()
+                .iter()
+                .filter(|fragment_id| {
+                    let Some(historical_fragment) = historical_fragments.get(fragment_id) else {
+                        return prune_historically_missing;
+                    };
+                    let Some(current_fragment) = current_fragments.get(fragment_id) else {
+                        return true;
+                    };
+                    let changed_files =
+                        fragment_field_paths(historical_fragment, &indexed_field_ids)
+                            != fragment_field_paths(current_fragment, &indexed_field_ids);
+                    let changed_overlays = prune_newer_overlays
+                        && current_fragment.overlays.iter().any(|overlay| {
+                            overlay.committed_version > version
+                                && overlay
+                                    .data_file
+                                    .fields
+                                    .iter()
+                                    .any(|field_id| indexed_field_ids.contains(field_id))
+                        });
+                    changed_files || changed_overlays
+                })
+                .collect::<Vec<_>>();
+            for fragment_id in stale_fragments {
+                segment.fragment_bitmap_mut().remove(fragment_id);
+            }
+        }
+    }
+    Ok(())
+}
+
 pub(crate) async fn build_index_metadata_from_segments(
     dataset: &Dataset,
     index_name: &str,
     field_id: i32,
-    segments: Vec<IndexSegment>,
+    mut segments: Vec<IndexSegment>,
 ) -> Result<Vec<IndexMetadata>> {
     if segments.is_empty() {
         return Err(Error::invalid_input(
@@ -145,6 +244,22 @@ pub(crate) async fn build_index_metadata_from_segments(
     let mut seen_segment_ids = HashSet::with_capacity(segments.len());
     let mut covered_fragments = RoaringBitmap::new();
     for segment in &segments {
+        if segment.dataset_version() > dataset.manifest.version {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: segment {} was built at future dataset version {} (current version {})",
+                segment.uuid(),
+                segment.dataset_version(),
+                dataset.manifest.version
+            )));
+        }
+        if segment.fields() != [field_id] {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: segment {} was built for fields {:?}, expected [{}]",
+                segment.uuid(),
+                segment.fields(),
+                field_id
+            )));
+        }
         if !seen_segment_ids.insert(segment.uuid()) {
             return Err(Error::invalid_input(format!(
                 "CreateIndex: duplicate segment uuid {} for index '{}'",
@@ -161,16 +276,19 @@ pub(crate) async fn build_index_metadata_from_segments(
         covered_fragments |= segment.fragment_bitmap().clone();
     }
 
+    prune_stale_segment_coverage(dataset, field_id, &mut segments, false, false).await?;
+
     let mut new_indices = Vec::with_capacity(segments.len());
     for segment in segments {
-        let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts();
+        let (uuid, fragment_bitmap, fields, index_details, index_version, dataset_version) =
+            segment.into_parts();
         let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails");
         if is_inverted_index {
             let metadata = IndexMetadata {
                 uuid,
                 name: index_name.to_string(),
-                fields: vec![field_id],
-                dataset_version: dataset.manifest.version,
+                fields: fields.clone(),
+                dataset_version,
                 fragment_bitmap: Some(fragment_bitmap.clone()),
                 index_details: Some(index_details.clone()),
                 index_version,
@@ -189,8 +307,8 @@ pub(crate) async fn build_index_metadata_from_segments(
         new_indices.push(IndexMetadata {
             uuid,
             name: index_name.to_string(),
-            fields: vec![field_id],
-            dataset_version: dataset.manifest.version,
+            fields,
+            dataset_version,
             fragment_bitmap: Some(fragment_bitmap),
             index_details: Some(index_details),
             index_version,
@@ -207,6 +325,227 @@ fn retain_committed_inverted_files(files: &mut Vec<IndexFile>) {
     files.retain(|file| !file.path.starts_with("staging/"));
 }
 
+async fn prewarm_opened_index(
+    index: Arc<dyn Index>,
+    options: Option<&PrewarmOptions>,
+) -> Result<()> {
+    match options {
+        None => index.prewarm().await,
+        Some(PrewarmOptions::Fts(fts_options)) => {
+            let inverted = index
+                .as_any()
+                .downcast_ref::<InvertedIndex>()
+                .ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "FTS prewarm options are only supported for inverted indices, got {:?}",
+                        index.index_type()
+                    ))
+                })?;
+            inverted.prewarm_with_options(fts_options).await
+        }
+        Some(_) => Err(Error::not_supported(
+            "unsupported prewarm options for this lance version".to_owned(),
+        )),
+    }
+}
+
+fn total_index_segment_size_bytes(indices: &[IndexMetadata]) -> Option<u64> {
+    let mut total = 0u64;
+    for index_meta in indices {
+        total += index_meta.total_size_bytes()?;
+    }
+    Some(total)
+}
+
+fn cache_size_delta(after: usize, before: usize) -> i64 {
+    after.saturating_sub(before) as i64 - before.saturating_sub(after) as i64
+}
+
+fn prewarm_options_fields(options: Option<&PrewarmOptions>) -> (&'static str, bool) {
+    match options {
+        None => ("default", false),
+        Some(PrewarmOptions::Fts(fts_options)) => ("fts", fts_options.with_position),
+        Some(_) => ("unsupported", false),
+    }
+}
+
+async fn prewarm_index_segments_by_metadata(
+    dataset: &Dataset,
+    name: &str,
+    indices: Vec<IndexMetadata>,
+    options: Option<&PrewarmOptions>,
+    available_segment_count: usize,
+    requested_segment_count: Option<usize>,
+) -> Result<()> {
+    let request_started = Instant::now();
+    let selected_segment_count = indices.len();
+    let selected_size_bytes = total_index_segment_size_bytes(&indices);
+    let (prewarm_options, fts_with_position) = prewarm_options_fields(options);
+    let cache_stats_before = dataset.session.index_cache_stats().await;
+    info!(
+        index_name = name,
+        selected_segment_count,
+        available_segment_count,
+        requested_segment_count = requested_segment_count.unwrap_or(0),
+        segment_filter = requested_segment_count.is_some(),
+        selected_size_bytes = selected_size_bytes.unwrap_or(0),
+        selected_size_bytes_known = selected_size_bytes.is_some(),
+        index_cache_entries_before = cache_stats_before.num_entries,
+        index_cache_size_bytes_before = cache_stats_before.size_bytes,
+        prewarm_options,
+        fts_with_position,
+        "prewarm index segments started"
+    );
+
+    let result = futures::future::try_join_all(indices.into_iter().map(|index_meta| async move {
+        let index_uuid = index_meta.uuid;
+        let size_bytes = index_meta.total_size_bytes();
+        let fragment_count = index_meta
+            .fragment_bitmap
+            .as_ref()
+            .map(|bitmap| bitmap.len());
+        let segment_started = Instant::now();
+        info!(
+            index_name = name,
+            %index_uuid,
+            index_version = index_meta.index_version,
+            dataset_version = index_meta.dataset_version,
+            fragment_count = fragment_count.unwrap_or(0),
+            fragment_count_known = fragment_count.is_some(),
+            size_bytes = size_bytes.unwrap_or(0),
+            size_bytes_known = size_bytes.is_some(),
+            "prewarm index segment started"
+        );
+
+        let index = match dataset
+            .open_generic_index(name, &index_uuid, &NoOpMetricsCollector)
+            .await
+        {
+            Ok(index) => {
+                info!(
+                    index_name = name,
+                    %index_uuid,
+                    index_type = ?index.index_type(),
+                    elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                    "opened index segment for prewarm"
+                );
+                index
+            }
+            Err(err) => {
+                warn!(
+                    index_name = name,
+                    %index_uuid,
+                    error = %err,
+                    elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                    "failed to open index segment for prewarm"
+                );
+                return Err(err);
+            }
+        };
+
+        if let Err(err) = prewarm_opened_index(index, options).await {
+            warn!(
+                index_name = name,
+                %index_uuid,
+                error = %err,
+                elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                "prewarm index segment failed"
+            );
+            return Err(err);
+        }
+
+        info!(
+            index_name = name,
+            %index_uuid,
+            elapsed_ms = segment_started.elapsed().as_millis() as u64,
+            "prewarm index segment finished"
+        );
+        Ok(())
+    }))
+    .await;
+
+    match result {
+        Ok(_) => {
+            let cache_stats_after = dataset.session.index_cache_stats().await;
+            info!(
+                index_name = name,
+                selected_segment_count,
+                index_cache_entries_after = cache_stats_after.num_entries,
+                index_cache_entries_delta = cache_size_delta(
+                    cache_stats_after.num_entries,
+                    cache_stats_before.num_entries,
+                ),
+                index_cache_size_bytes_after = cache_stats_after.size_bytes,
+                index_cache_size_bytes_delta =
+                    cache_size_delta(cache_stats_after.size_bytes, cache_stats_before.size_bytes,),
+                elapsed_ms = request_started.elapsed().as_millis() as u64,
+                "prewarm index segments finished"
+            );
+            Ok(())
+        }
+        Err(err) => {
+            let cache_stats_after = dataset.session.index_cache_stats().await;
+            warn!(
+                index_name = name,
+                selected_segment_count,
+                index_cache_entries_after = cache_stats_after.num_entries,
+                index_cache_entries_delta = cache_size_delta(
+                    cache_stats_after.num_entries,
+                    cache_stats_before.num_entries,
+                ),
+                index_cache_size_bytes_after = cache_stats_after.size_bytes,
+                index_cache_size_bytes_delta = cache_size_delta(
+                    cache_stats_after.size_bytes,
+                    cache_stats_before.size_bytes,
+                ),
+                error = %err,
+                elapsed_ms = request_started.elapsed().as_millis() as u64,
+                "prewarm index segments failed"
+            );
+            Err(err)
+        }
+    }
+}
+
+fn filter_index_segments_by_ids(
+    name: &str,
+    indices: Vec<IndexMetadata>,
+    segment_ids: &[Uuid],
+) -> Result<Vec<IndexMetadata>> {
+    if segment_ids.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let requested = segment_ids.iter().copied().collect::<HashSet<_>>();
+    let mut matched = HashSet::new();
+    let filtered = indices
+        .into_iter()
+        .filter(|index_meta| {
+            if requested.contains(&index_meta.uuid) {
+                matched.insert(index_meta.uuid);
+                true
+            } else {
+                false
+            }
+        })
+        .collect::<Vec<_>>();
+
+    if matched.len() != requested.len() {
+        let mut missing = requested
+            .difference(&matched)
+            .map(ToString::to_string)
+            .collect::<Vec<_>>();
+        missing.sort();
+        return Err(Error::index_not_found(format!(
+            "name={}, segment_ids=[{}]",
+            name,
+            missing.join(", ")
+        )));
+    }
+
+    Ok(filtered)
+}
+
 fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> {
     let mut type_url = None::<&str>;
     for segment in segments {
@@ -265,6 +604,13 @@ fn segment_has_bitmap_details(segment: &IndexMetadata) -> bool {
         .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails"))
 }
 
+fn segment_has_bloomfilter_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("BloomFilterIndexDetails"))
+}
+
 /// Detect BTree segments, preserving a legacy pre-details fallback.
 fn segment_has_btree_details(segment: &IndexMetadata) -> bool {
     segment.index_details.as_ref().map_or_else(
@@ -299,6 +645,13 @@ fn segment_has_label_list_details(segment: &IndexMetadata) -> bool {
         .is_some_and(|details| details.type_url.ends_with("LabelListIndexDetails"))
 }
 
+fn segment_has_rtree_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("RTreeIndexDetails"))
+}
+
 // Cache keys for different index types
 #[derive(Debug, Clone)]
 pub(crate) struct LegacyVectorIndexCacheKey<'a> {
@@ -605,7 +958,7 @@ pub(crate) async fn remap_index(
                 )
                 .unwrap(),
                 index_version,
-                files,
+                files: table_files_to_index(files),
             }
         }
         _ => {
@@ -621,7 +974,7 @@ pub(crate) async fn remap_index(
         new_id,
         index_details: created_index.index_details,
         index_version: created_index.index_version,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
     }))
 }
 
@@ -781,10 +1134,15 @@ impl IndexDescriptionImpl {
         let mut missing_fragment_refs = 0u64;
 
         for shard in &segments {
-            let fragment_bitmap = shard
-            .fragment_bitmap
-            .as_ref()
-            .ok_or_else(|| Error::index("Fragment bitmap is required for index description.  This index must be retrained to support this method.".to_string()))?;
+            let Some(fragment_bitmap) = shard.fragment_bitmap.as_ref() else {
+                // A system index (e.g. __mem_wal) indexes no fragments, so a
+                // missing bitmap means zero indexed rows. For a data index it
+                // means unknown coverage — reject rather than fabricate a count.
+                if is_system_index(shard) {
+                    continue;
+                }
+                return Err(Error::index("Fragment bitmap is required for index description.  This index must be retrained to support this method.".to_string()));
+            };
 
             indexed_fragment_refs += fragment_bitmap.len();
             for fragment_id in fragment_bitmap.iter() {
@@ -974,49 +1332,70 @@ impl DatasetIndexExt for Dataset {
             return Err(Error::index_not_found(format!("name={}", name)));
         }
 
-        for index_meta in indices {
-            let index = self
-                .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector)
-                .await?;
-            index.prewarm().await?;
+        let available_segment_count = indices.len();
+        prewarm_index_segments_by_metadata(self, name, indices, None, available_segment_count, None)
+            .await
+    }
+
+    async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            return Err(Error::index_not_found(format!("name={}", name)));
         }
 
-        Ok(())
+        let available_segment_count = indices.len();
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            Some(options),
+            available_segment_count,
+            None,
+        )
+        .await
     }
 
-    async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> {
+    async fn prewarm_index_segments(&self, name: &str, segment_ids: &[Uuid]) -> Result<()> {
         let indices = self.load_indices_by_name(name).await?;
         if indices.is_empty() {
             return Err(Error::index_not_found(format!("name={}", name)));
         }
+        let available_segment_count = indices.len();
+        let indices = filter_index_segments_by_ids(name, indices, segment_ids)?;
 
-        for index_meta in indices {
-            let index = self
-                .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector)
-                .await?;
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            None,
+            available_segment_count,
+            Some(segment_ids.len()),
+        )
+        .await
+    }
 
-            match options {
-                PrewarmOptions::Fts(fts_options) => {
-                    let inverted = index
-                        .as_any()
-                        .downcast_ref::<InvertedIndex>()
-                        .ok_or_else(|| {
-                            Error::invalid_input(format!(
-                                "FTS prewarm options are only supported for inverted indices, got {:?}",
-                                index.index_type()
-                            ))
-                        })?;
-                    inverted.prewarm_with_options(fts_options).await?;
-                }
-                _ => {
-                    return Err(Error::not_supported(
-                        "unsupported prewarm options for this lance version".to_owned(),
-                    ));
-                }
-            }
+    async fn prewarm_index_segments_with_options(
+        &self,
+        name: &str,
+        segment_ids: &[Uuid],
+        options: &PrewarmOptions,
+    ) -> Result<()> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            return Err(Error::index_not_found(format!("name={}", name)));
         }
+        let available_segment_count = indices.len();
+        let indices = filter_index_segments_by_ids(name, indices, segment_ids)?;
 
-        Ok(())
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            Some(options),
+            available_segment_count,
+            Some(segment_ids.len()),
+        )
+        .await
     }
 
     async fn describe_indices<'a, 'b>(
@@ -1066,10 +1445,11 @@ impl DatasetIndexExt for Dataset {
     async fn load_indices(&self) -> Result<Arc<Vec<IndexMetadata>>> {
         let metadata_key = IndexMetadataKey {
             version: self.version().version,
+            store_identity: &self.object_store.store_prefix,
         };
-        let mut indices = match self.index_cache.get_with_key(&metadata_key).await {
-            Some(indices) => indices,
-            None => {
+        let mut indices = self
+            .index_cache
+            .get_or_insert_with_key(metadata_key, || async {
                 let mut loaded_indices = read_manifest_indexes(
                     &self.object_store,
                     &self.manifest_location,
@@ -1077,13 +1457,9 @@ impl DatasetIndexExt for Dataset {
                 )
                 .await?;
                 retain_supported_indices(&mut loaded_indices);
-                let loaded_indices = Arc::new(loaded_indices);
-                self.index_cache
-                    .insert_with_key(&metadata_key, loaded_indices.clone())
-                    .await;
-                loaded_indices
-            }
-        };
+                Ok(loaded_indices)
+            })
+            .await?;
 
         // Infer details for legacy vector indices (once per index name, concurrently).
         // This may run on indices that were opportunistically cached during Dataset::open
@@ -1127,9 +1503,23 @@ impl DatasetIndexExt for Dataset {
 
     async fn merge_existing_index_segments(
         &self,
-        source_segments: Vec<IndexMetadata>,
+        mut source_segments: Vec<IndexMetadata>,
     ) -> Result<IndexMetadata> {
         validate_segment_metadata("uncommitted", &source_segments)?;
+        if let Some(segment) = source_segments
+            .iter()
+            .find(|segment| segment.dataset_version > self.manifest.version)
+        {
+            return Err(Error::invalid_input(format!(
+                "merge_existing_index_segments: segment {} was built at future dataset version {} (current version {})",
+                segment.uuid, segment.dataset_version, self.manifest.version
+            )));
+        }
+        let source_dataset_version = source_segments
+            .iter()
+            .map(|segment| segment.dataset_version)
+            .min()
+            .unwrap_or(self.manifest.version);
         let field_id = *source_segments[0].fields.first().ok_or_else(|| {
             Error::invalid_input(format!(
                 "CreateIndex: segment {} is missing field ids",
@@ -1147,17 +1537,21 @@ impl DatasetIndexExt for Dataset {
         let all_vector = source_segments.iter().all(segment_has_vector_details);
         let all_inverted = source_segments.iter().all(segment_has_inverted_details);
         let all_bitmap = source_segments.iter().all(segment_has_bitmap_details);
+        let all_bloomfilter = source_segments.iter().all(segment_has_bloomfilter_details);
         let all_btree = source_segments.iter().all(segment_has_btree_details);
         let all_fmindex = source_segments.iter().all(segment_has_fmindex_details);
         let all_zonemap = source_segments.iter().all(segment_has_zonemap_details);
         let all_label_list = source_segments.iter().all(segment_has_label_list_details);
+        let all_rtree = source_segments.iter().all(segment_has_rtree_details);
         if !all_vector
             && !all_inverted
             && !all_bitmap
+            && !all_bloomfilter
             && !all_btree
             && !all_fmindex
             && !all_zonemap
             && !all_label_list
+            && !all_rtree
         {
             return Err(Error::invalid_input(
                 "merge_existing_index_segments requires all segments to have the same supported index type"
@@ -1165,6 +1559,21 @@ impl DatasetIndexExt for Dataset {
             ));
         }
 
+        let merged_dataset_version = if all_rtree {
+            let mut source_coverage = source_segments
+                .iter()
+                .cloned()
+                .map(IntoIndexSegment::into_index_segment)
+                .collect::<Result<Vec<_>>>()?;
+            prune_stale_segment_coverage(self, field_id, &mut source_coverage, true, true).await?;
+            for (source, coverage) in source_segments.iter_mut().zip(source_coverage) {
+                source.fragment_bitmap = Some(coverage.fragment_bitmap().clone());
+            }
+            self.manifest.version
+        } else {
+            source_dataset_version
+        };
+
         let mut merged_segment = if all_vector {
             crate::index::vector::ivf::merge_segments(
                 self.object_store.as_ref(),
@@ -1178,14 +1587,25 @@ impl DatasetIndexExt for Dataset {
             crate::index::scalar::fmindex::merge_segments(self, source_segments).await?
         } else if all_bitmap {
             crate::index::scalar::bitmap::merge_segments(self, source_segments).await?
+        } else if all_bloomfilter {
+            crate::index::scalar::bloomfilter::merge_segments(self, source_segments).await?
         } else if all_label_list {
             crate::index::scalar::label_list::merge_segments(self, source_segments).await?
         } else if all_zonemap {
             crate::index::scalar::zonemap::merge_segments(self, source_segments).await?
+        } else if all_rtree {
+            #[cfg(feature = "geo")]
+            {
+                crate::index::scalar::rtree::merge_segments(self, source_segments).await?
+            }
+            #[cfg(not(feature = "geo"))]
+            return Err(Error::not_supported(
+                "RTree segment merge requires the `geo` feature".to_string(),
+            ));
         } else {
             crate::index::scalar::btree::merge_segments(self, source_segments).await?
         };
-        merged_segment.dataset_version = self.manifest.version;
+        merged_segment.dataset_version = merged_dataset_version;
         merged_segment.fields = vec![field_id];
         Ok(merged_segment)
     }
@@ -1566,7 +1986,7 @@ async fn index_statistics_frag_reuse(ds: &Dataset) -> Result<String> {
         .open_frag_reuse_index(&NoOpMetricsCollector)
         .await?
         .expect("FragmentReuse index does not exist");
-    serialize_index_statistics(&index.statistics()?)
+    serialize_index_statistics(&FragReuseIndexHandle(index).statistics()?)
 }
 
 async fn index_statistics_mem_wal(ds: &Dataset) -> Result<String> {
@@ -1574,7 +1994,7 @@ async fn index_statistics_mem_wal(ds: &Dataset) -> Result<String> {
         .open_mem_wal_index(&NoOpMetricsCollector)
         .await?
         .expect("MemWal index does not exist");
-    serialize_index_statistics(&index.statistics()?)
+    serialize_index_statistics(&MemWalIndexHandle(index).statistics()?)
 }
 
 async fn index_statistics_scalar(
@@ -1834,7 +2254,7 @@ impl DatasetIndexInternalExt for Dataset {
 
         let frag_reuse_cache_key = FragReuseIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
         if let Some(index) = self.index_cache.get_with_key(&frag_reuse_cache_key).await {
-            return Ok(index.as_index());
+            return Ok(Arc::new(FragReuseIndexHandle(index)).as_index());
         }
 
         // Sometimes we want to open an index and we don't care if it is a scalar or vector index.
@@ -2636,7 +3056,7 @@ mod tests {
     use crate::session::Session;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, copy_test_data_to_tmp};
     use arrow::array::AsArray;
-    use arrow::datatypes::{Float32Type, Int32Type};
+    use arrow::datatypes::{Float32Type, Int32Type, Int64Type};
     use arrow_array::Int32Array;
     use arrow_array::{
         FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray,
@@ -2647,8 +3067,12 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datagen::gen_batch;
     use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array};
-    use lance_index::pbold::BTreeIndexDetails;
+    use lance_index::pbold::{BTreeIndexDetails, InvertedIndexDetails};
     use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME;
+    use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery};
+    use lance_index::scalar::inverted::{
+        INVERTED_INDEX_VERSION_V1, INVERTED_INDEX_VERSION_V2, INVERTED_INDEX_VERSION_V3,
+    };
     use lance_index::scalar::{
         BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
     };
@@ -2722,12 +3146,14 @@ mod tests {
                 .as_ref()
                 .expect("test segment metadata should have fragment coverage")
                 .iter(),
+            metadata.fields.iter().copied(),
             metadata
                 .index_details
                 .as_ref()
                 .expect("test segment metadata should have index details")
                 .clone(),
             metadata.index_version,
+            metadata.dataset_version,
         )
     }
 
@@ -3644,6 +4070,53 @@ mod tests {
         assert_eq!(stats["num_indices"], 1);
     }
 
+    #[rstest]
+    #[case::v1("v3.0.1/fts_v1", INVERTED_INDEX_VERSION_V1)]
+    #[case::v2("v4.0.1/fts_v2", INVERTED_INDEX_VERSION_V2)]
+    #[tokio::test]
+    async fn test_read_fts_format_fixture(
+        #[case] fixture_path: &str,
+        #[case] expected_version: u32,
+    ) {
+        async fn search_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec<i64> {
+            let result = dataset
+                .scan()
+                .project(&["id"])
+                .unwrap()
+                .full_text_search(query)
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let mut ids = result["id"].as_primitive::<Int64Type>().values().to_vec();
+            ids.sort_unstable();
+            ids
+        }
+
+        let test_dir = copy_test_data_to_tmp(fixture_path).unwrap();
+        let dataset = Dataset::open(&test_dir.path_str()).await.unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_version, expected_version as i32);
+
+        let match_ids = search_ids(
+            &dataset,
+            FullTextSearchQuery::new("compatibility".to_string()),
+        )
+        .await;
+        assert_eq!(match_ids, (0..300).collect::<Vec<_>>());
+
+        let phrase =
+            PhraseQuery::new("lance database".to_string()).with_column(Some("text".to_string()));
+        let phrase_ids = search_ids(
+            &dataset,
+            FullTextSearchQuery::new_query(FtsQuery::Phrase(phrase)),
+        )
+        .await;
+        assert_eq!(phrase_ids, (0..300).step_by(3).collect::<Vec<_>>());
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_optimize_fts(#[values(false, true)] with_position: bool) {
@@ -4102,6 +4575,48 @@ mod tests {
         assert_eq!(indices2.len(), 1);
     }
 
+    #[tokio::test]
+    async fn test_load_indices_singleflights_concurrent_cache_misses() {
+        let session = Arc::new(Session::default());
+        let write_params = WriteParams {
+            session: Some(session.clone()),
+            ..Default::default()
+        };
+        let test_dir = TempStrDir::default();
+        let schema = Arc::new(Schema::new(vec![Field::new("tag", DataType::Utf8, false)]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(StringArray::from(vec!["a", "b", "c"]))],
+        )
+        .unwrap();
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch)], schema),
+            &test_dir,
+            Some(write_params),
+        )
+        .await
+        .unwrap();
+        dataset
+            .create_index(
+                &["tag"],
+                IndexType::Bitmap,
+                None,
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        session.index_cache.clear().await;
+        let before = session.index_cache_stats().await;
+        let results = futures::future::join_all((0..32).map(|_| dataset.load_indices())).await;
+        assert!(results.iter().all(|result| result.is_ok()));
+        let after = session.index_cache_stats().await;
+
+        assert_eq!(after.misses - before.misses, 1);
+        assert_eq!(after.hits - before.hits, 31);
+    }
+
     #[tokio::test]
     async fn test_remap_empty() {
         let data = gen_batch()
@@ -4488,6 +5003,7 @@ mod tests {
         // We commit by doing a delete("false") after replacing the cached indices.
         let metadata_key = crate::session::index_caches::IndexMetadataKey {
             version: dataset.version().version,
+            store_identity: &dataset.object_store.store_prefix,
         };
         dataset
             .index_cache
@@ -4715,6 +5231,83 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case::block_size_128(128)]
+    #[case::block_size_256(256)]
+    #[tokio::test]
+    async fn test_optimize_empty_code_fts_index_preserves_params(#[case] block_size: usize) {
+        let dir = TempStrDir::default();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values([0, 1])),
+                Arc::new(StringArray::from_iter_values([
+                    "fn GetUserEmail() {}",
+                    "fn ParseConfig() {}",
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(reader, &dir, None).await.unwrap();
+
+        let params = InvertedIndexParams::default()
+            .analyzer("code")
+            .unwrap()
+            .block_size(block_size)
+            .unwrap()
+            .split_identifiers(true)
+            .preserve_original(false);
+        dataset
+            .create_index_builder(&["code"], IndexType::Inverted, &params)
+            .name("code_idx".to_string())
+            .train(false)
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32);
+
+        dataset.optimize_indices(&Default::default()).await.unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32);
+        let details = indices[0]
+            .index_details
+            .as_ref()
+            .expect("optimized FTS index should retain index details")
+            .to_msg::<InvertedIndexDetails>()
+            .unwrap();
+        assert_eq!(details.base_tokenizer.as_deref(), Some("code"));
+        assert_eq!(details.block_size, Some(block_size as u32));
+        let code_config = details
+            .code_config
+            .expect("optimized code FTS index should retain code configuration");
+        assert!(code_config.split_identifiers);
+        assert_eq!(code_config.preserve_original, Some(false));
+
+        let stats: serde_json::Value =
+            serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap();
+        assert_eq!(stats["num_unindexed_rows"], 0);
+
+        let result = dataset
+            .scan()
+            .project(&["id"])
+            .unwrap()
+            .full_text_search(FullTextSearchQuery::new("email".to_string()))
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(result["id"].as_primitive::<Int32Type>().values(), &[0]);
+    }
+
     /// Helper function to check if an index is being used in a query plan
     fn assert_index_usage(plan: &str, column_name: &str, should_use_index: bool, context: &str) {
         let index_used = if column_name == "text" {
@@ -6736,6 +7329,83 @@ mod tests {
         assert!(err.to_string().contains("at least one index segment"));
     }
 
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_wrong_field_provenance() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
+
+        let vector_field_id = dataset.schema().field("vector").unwrap().id;
+        let mut metadata = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            vector_field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"segment",
+        )
+        .await;
+        metadata.fields = vec![dataset.schema().field("id").unwrap().id];
+
+        let error = dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&metadata)],
+            )
+            .await
+            .unwrap_err();
+        assert!(error.to_string().contains("was built for fields"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_future_dataset_version() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
+
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let mut metadata = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"segment",
+        )
+        .await;
+        metadata.dataset_version = dataset.manifest.version + 1;
+
+        let error = dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&metadata)],
+            )
+            .await
+            .unwrap_err();
+        assert!(error.to_string().contains("future dataset version"));
+    }
+
     #[tokio::test]
     async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() {
         use lance_datagen::{BatchCount, RowCount, array};
diff --git a/rust/lance/src/index/api.rs b/rust/lance/src/index/api.rs
index f856e9004f3..431c9fb2330 100644
--- a/rust/lance/src/index/api.rs
+++ b/rust/lance/src/index/api.rs
@@ -15,37 +15,45 @@ use crate::{Error, Result};
 /// A single physical segment of a logical index.
 ///
 /// Each segment is stored independently and will become one manifest entry when committed.
-/// The logical index identity (name / target column / dataset version) is provided separately
-/// by the commit API.
+/// The logical index name is provided separately by the commit API, while physical field and
+/// dataset-version provenance travel with the segment.
 #[derive(Debug, Clone, PartialEq)]
 pub struct IndexSegment {
     /// Unique ID of the physical segment.
     uuid: Uuid,
     /// The fragments covered by this segment.
     fragment_bitmap: RoaringBitmap,
+    /// Field IDs whose physical values are encoded in this segment.
+    fields: Vec<i32>,
     /// Metadata specific to the index type.
     index_details: Arc<prost_types::Any>,
     /// The on-disk index version for this segment.
     index_version: i32,
+    /// Dataset version at which this segment's physical contents were built.
+    dataset_version: u64,
 }
 
 impl IndexSegment {
-    /// Create a fully described segment with the given UUID, fragment coverage, and index
-    /// metadata.
-    pub fn new<I>(
+    /// Create a fully described segment with its physical build provenance.
+    pub fn new<I, F>(
         uuid: Uuid,
         fragment_bitmap: I,
+        fields: F,
         index_details: Arc<prost_types::Any>,
         index_version: i32,
+        dataset_version: u64,
     ) -> Self
     where
         I: IntoIterator<Item = u32>,
+        F: IntoIterator<Item = i32>,
     {
         Self {
             uuid,
             fragment_bitmap: fragment_bitmap.into_iter().collect(),
+            fields: fields.into_iter().collect(),
             index_details,
             index_version,
+            dataset_version,
         }
     }
 
@@ -59,6 +67,15 @@ impl IndexSegment {
         &self.fragment_bitmap
     }
 
+    pub(crate) fn fragment_bitmap_mut(&mut self) -> &mut RoaringBitmap {
+        &mut self.fragment_bitmap
+    }
+
+    /// Return the field IDs whose values are encoded in this segment.
+    pub fn fields(&self) -> &[i32] {
+        &self.fields
+    }
+
     /// Return the serialized index details for this segment.
     pub fn index_details(&self) -> &Arc<prost_types::Any> {
         &self.index_details
@@ -69,13 +86,29 @@ impl IndexSegment {
         self.index_version
     }
 
+    /// Return the source dataset version for this segment.
+    pub fn dataset_version(&self) -> u64 {
+        self.dataset_version
+    }
+
     /// Consume the segment and return its component parts.
-    pub fn into_parts(self) -> (Uuid, RoaringBitmap, Arc<prost_types::Any>, i32) {
+    pub fn into_parts(
+        self,
+    ) -> (
+        Uuid,
+        RoaringBitmap,
+        Vec<i32>,
+        Arc<prost_types::Any>,
+        i32,
+        u64,
+    ) {
         (
             self.uuid,
             self.fragment_bitmap,
+            self.fields,
             self.index_details,
             self.index_version,
+            self.dataset_version,
         )
     }
 }
@@ -110,8 +143,10 @@ impl IntoIndexSegment for IndexMetadata {
         Ok(IndexSegment::new(
             self.uuid,
             fragment_bitmap.iter(),
+            self.fields,
             index_details,
             self.index_version,
+            self.dataset_version,
         ))
     }
 }
@@ -167,6 +202,25 @@ pub trait DatasetIndexExt {
         ))
     }
 
+    /// Prewarm selected physical segments of an index by name.
+    async fn prewarm_index_segments(&self, _name: &str, _segment_ids: &[Uuid]) -> Result<()> {
+        Err(Error::not_supported(
+            "segment-level prewarm is not supported by this dataset implementation".to_owned(),
+        ))
+    }
+
+    /// Prewarm selected physical segments of an index by name with additional options.
+    async fn prewarm_index_segments_with_options(
+        &self,
+        _name: &str,
+        _segment_ids: &[Uuid],
+        _options: &PrewarmOptions,
+    ) -> Result<()> {
+        Err(Error::not_supported(
+            "prewarm options are not supported by this dataset implementation".to_owned(),
+        ))
+    }
+
     /// Read all indices of this Dataset version.
     ///
     /// The indices are lazy loaded and cached in memory within the `Dataset` instance.
@@ -254,3 +308,31 @@ pub trait DatasetIndexExt {
         with_vector: bool,
     ) -> Result<SendableRecordBatchStream>;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_index_metadata_conversion_preserves_provenance() {
+        let metadata = IndexMetadata {
+            uuid: Uuid::new_v4(),
+            name: "test".to_string(),
+            fields: vec![3, 7],
+            dataset_version: 42,
+            fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2])),
+            index_details: Some(Arc::new(prost_types::Any {
+                type_url: "example.IndexDetails".to_string(),
+                value: vec![1, 2, 3],
+            })),
+            index_version: 5,
+            created_at: None,
+            base_id: None,
+            files: None,
+        };
+
+        let segment = metadata.into_index_segment().unwrap();
+        assert_eq!(segment.fields(), [3, 7]);
+        assert_eq!(segment.dataset_version(), 42);
+    }
+}
diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs
index d3ecde030c1..9dfc61894b2 100644
--- a/rust/lance/src/index/append.rs
+++ b/rust/lance/src/index/append.rs
@@ -5,16 +5,22 @@ use std::sync::Arc;
 
 use futures::{FutureExt, TryStreamExt};
 use lance_core::{Error, Result};
+use lance_file::reader::FileReaderOptions;
 use lance_index::{
     INDEX_FILE_NAME, IndexType,
     metrics::NoOpMetricsCollector,
     optimize::OptimizeOptions,
     progress::NoopIndexBuildProgress,
     scalar::{
-        CreatedIndex, OldIndexDataFilter, ScalarIndex, inverted::InvertedIndex,
+        CreatedIndex, OldIndexDataFilter, ScalarIndex, index_files_to_table,
+        inverted::InvertedIndex,
         lance_format::LanceIndexStore,
+        seed::{FragmentSeed, SEED_META_KEY_PREFIX},
+        table_files_to_index,
     },
 };
+use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+use lance_io::utils::CachedFileSize;
 use lance_select::{RowAddrTreeMap, RowSetOps};
 use lance_table::format::{Fragment, IndexMetadata};
 use roaring::RoaringBitmap;
@@ -26,7 +32,7 @@ use super::vector::ivf::{optimize_vector_indices, select_segment_for_single_reba
 use crate::dataset::Dataset;
 use crate::dataset::index::LanceIndexStoreExt;
 use crate::dataset::rowids::load_row_id_sequences;
-use crate::index::scalar::load_training_data;
+use crate::index::scalar::{IndexDetails, fetch_index_details, load_training_data};
 use crate::index::vector_index_details_default;
 
 #[derive(Debug, Clone)]
@@ -175,6 +181,88 @@ pub async fn build_per_segment_filters(
     Ok((effective_union, filters))
 }
 
+/// Attempt to read seed buffers for `column_name` from `fragments`' data files.
+///
+/// Returns `Some(vec)` only if every fragment has a seed entry; returns `None`
+/// if any fragment is missing a seed or its data file cannot be opened.
+/// Index-type-specific validation (e.g. `rows_per_zone` checks) is left to the
+/// caller via [`FragmentSeed::metadata_value`].
+async fn try_harvest_seeds(
+    dataset: &Dataset,
+    fragments: &[Fragment],
+    column_name: &str,
+) -> Result<Option<Vec<FragmentSeed>>> {
+    if fragments.is_empty() {
+        return Ok(Some(Vec::new()));
+    }
+
+    let meta_key = format!("{}{}", SEED_META_KEY_PREFIX, column_name);
+    let mut seeds = Vec::with_capacity(fragments.len());
+
+    let scheduler = ScanScheduler::new(
+        dataset.object_store.clone(),
+        SchedulerConfig::max_bandwidth(&dataset.object_store),
+    );
+
+    for fragment in fragments {
+        let Some(data_file) = fragment.files.first() else {
+            return Ok(None);
+        };
+
+        let path = dataset
+            .base
+            .clone()
+            .join(crate::dataset::DATA_DIR)
+            .join(data_file.path.as_str());
+        let Ok(file_scheduler) = scheduler.open_file(&path, &CachedFileSize::unknown()).await
+        else {
+            return Ok(None);
+        };
+
+        let Ok(reader) = lance_file::reader::FileReader::try_open(
+            file_scheduler,
+            None,
+            Default::default(),
+            &dataset.metadata_cache.file_metadata_cache(&path),
+            FileReaderOptions::default(),
+        )
+        .await
+        else {
+            return Ok(None);
+        };
+
+        let Some(meta_value) = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .get(&meta_key)
+            .cloned()
+        else {
+            return Ok(None);
+        };
+
+        // The buf_index is always the portion before the first ':'.
+        let Some(buf_index_str) = meta_value.split(':').next() else {
+            return Ok(None);
+        };
+        let Ok(buf_index) = buf_index_str.parse::<u32>() else {
+            return Ok(None);
+        };
+
+        let Ok(bytes) = reader.read_global_buffer(buf_index).await else {
+            return Ok(None);
+        };
+
+        seeds.push(FragmentSeed {
+            fragment_id: fragment.id,
+            bytes,
+            metadata_value: meta_value,
+        });
+    }
+
+    Ok(Some(seeds))
+}
+
 async fn load_unindexed_training_data(
     dataset: &Dataset,
     field_path: &str,
@@ -356,38 +444,67 @@ async fn merge_scalar_indices<'a>(
         )
         .await?
     } else {
-        let new_data_stream =
-            load_unindexed_training_data(dataset.as_ref(), field_path, &update_criteria, unindexed)
-                .await?;
         let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?;
 
-        match index_type {
-            IndexType::BTree => {
-                let (_, old_data_filters) =
-                    build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?;
-                crate::index::scalar::btree::open_and_merge_segments(
-                    dataset.as_ref(),
-                    field_path,
-                    &selected_old_indices,
-                    new_data_stream,
-                    &new_store,
-                    &old_data_filters,
-                )
-                .await?
+        // Try a seed-based update before falling back to a full column scan.
+        // Seeds are only available if every unindexed fragment had a seed buffer
+        // written during its data file write, and the plugin validates that the
+        // seeds are compatible with the current index configuration.
+        let index_details =
+            fetch_index_details(dataset.as_ref(), field_path, reference_idx).await?;
+        let details = IndexDetails(index_details.clone());
+        let plugin = details.get_plugin()?;
+        // Only open data files looking for seeds when the plugin confirms this
+        // index type and configuration can actually produce them.
+        let maybe_created = if plugin.might_use_seeds(&index_details) {
+            if let Some(seeds) = try_harvest_seeds(dataset.as_ref(), unindexed, column_name).await?
+            {
+                plugin
+                    .update_from_seeds(seeds, reference_index.clone(), &index_details, &new_store)
+                    .await?
+            } else {
+                None
             }
-            // NOTE: IndexType::Inverted never reaches here -- it is handled by the
-            // dedicated arm in merge_indices_with_unindexed_frags before this
-            // function is called.
-            _ => {
-                let old_data_filter = build_old_data_filter(
-                    dataset.as_ref(),
-                    &effective_old_frags,
-                    &deleted_old_frags,
-                )
-                .await?;
-                reference_index
-                    .update(new_data_stream, &new_store, old_data_filter)
+        } else {
+            None
+        };
+
+        if let Some(created) = maybe_created {
+            created
+        } else {
+            let new_data_stream = load_unindexed_training_data(
+                dataset.as_ref(),
+                field_path,
+                &update_criteria,
+                unindexed,
+            )
+            .await?;
+
+            match index_type {
+                IndexType::BTree => {
+                    let (_, old_data_filters) =
+                        build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?;
+                    crate::index::scalar::btree::open_and_merge_segments(
+                        dataset.as_ref(),
+                        field_path,
+                        &selected_old_indices,
+                        new_data_stream,
+                        &new_store,
+                        &old_data_filters,
+                    )
                     .await?
+                }
+                _ => {
+                    let old_data_filter = build_old_data_filter(
+                        dataset.as_ref(),
+                        &effective_old_frags,
+                        &deleted_old_frags,
+                    )
+                    .await?;
+                    reference_index
+                        .update(new_data_stream, &new_store, old_data_filter)
+                        .await?
+                }
             }
         }
     };
@@ -592,7 +709,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 CreatedIndex {
                     index_details: vector_index_details_default(),
                     index_version: lance_index::IndexType::Vector.version() as u32,
-                    files,
+                    files: table_files_to_index(files),
                 },
             ))
         } else {
@@ -655,7 +772,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     // index_version <= our max supported version, so we can safely
                     // write the current library's version for this index type.
                     index_version: lance_index::IndexType::Vector.version() as u32,
-                    files,
+                    files: table_files_to_index(files),
                 },
             ))
         }
@@ -734,7 +851,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                         new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(),
                         new_index_version: created_index.index_version as i32,
                         new_index_details: created_index.index_details,
-                        files: created_index.files,
+                        files: index_files_to_table(created_index.files),
                     }));
                 }
 
@@ -850,7 +967,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
         new_fragment_bitmap,
         new_index_version: created_index.index_version as i32,
         new_index_details: created_index.index_details,
-        files: created_index.files,
+        files: index_files_to_table(created_index.files),
     }))
 }
 
@@ -863,8 +980,10 @@ mod tests {
     use arrow::datatypes::{Float32Type, UInt32Type};
     use arrow_array::cast::AsArray;
     use arrow_array::{
-        FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt32Array,
+        ArrayRef, FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray,
+        UInt32Array,
     };
+    use arrow_buffer::{BooleanBufferBuilder, NullBuffer};
     use arrow_schema::{DataType, Field, Schema};
     use futures::TryStreamExt;
     use lance_arrow::FixedSizeListArrayExt;
@@ -1005,6 +1124,118 @@ mod tests {
         assert_eq!(num_rows, 2000);
     }
 
+    #[tokio::test]
+    async fn test_optimize_append_preserves_case_sensitive_nullable_vector_column() {
+        const DIM: usize = 64;
+        const ROWS: usize = 1000;
+
+        fn make_vectors(rows: usize, dim: usize, include_null: bool) -> FixedSizeListArray {
+            if include_null {
+                let mut nulls_builder = BooleanBufferBuilder::new(rows);
+                for row_idx in 0..rows {
+                    nulls_builder.append(row_idx != 0);
+                }
+                let nulls = NullBuffer::new(nulls_builder.finish());
+                FixedSizeListArray::try_new(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dim as i32,
+                    Arc::new(generate_random_array(rows * dim)),
+                    Some(nulls),
+                )
+                .unwrap()
+            } else {
+                FixedSizeListArray::try_new_from_values(
+                    generate_random_array(rows * dim),
+                    dim as i32,
+                )
+                .unwrap()
+            }
+        }
+
+        fn make_batch(
+            schema: Arc<Schema>,
+            start_id: u32,
+            vectors: Arc<FixedSizeListArray>,
+        ) -> RecordBatch {
+            let columns: Vec<ArrayRef> = vec![
+                Arc::new(UInt32Array::from_iter_values(
+                    start_id..start_id + ROWS as u32,
+                )) as ArrayRef,
+                vectors as ArrayRef,
+            ];
+            RecordBatch::try_new(schema, columns).unwrap()
+        }
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let vector_type = DataType::FixedSizeList(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            DIM as i32,
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new("VECTOR", vector_type, true),
+        ]));
+
+        let initial_vectors = Arc::new(make_vectors(ROWS, DIM, false));
+        let initial_batch = make_batch(schema.clone(), 0, initial_vectors);
+        let batches = RecordBatchIterator::new(std::iter::once(Ok(initial_batch)), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::with_ivf_pq_params(
+            MetricType::L2,
+            IvfBuildParams::new(2),
+            PQBuildParams {
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
+        dataset
+            .create_index(&["VECTOR"], IndexType::Vector, None, &params, true)
+            .await
+            .unwrap();
+
+        let appended_vectors = Arc::new(make_vectors(ROWS, DIM, true));
+        let query = appended_vectors.value(5);
+        let appended_batch = make_batch(schema.clone(), ROWS as u32, appended_vectors);
+        let batches = RecordBatchIterator::new(std::iter::once(Ok(appended_batch)), schema);
+        dataset.append(batches, None).await.unwrap();
+
+        let index_name = dataset.load_indices().await.unwrap()[0].name.clone();
+        assert!(
+            !dataset
+                .unindexed_fragments(&index_name)
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        assert!(
+            dataset
+                .unindexed_fragments(&index_name)
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        let mut scanner = dataset.scan();
+        scanner
+            .nearest("VECTOR", query.as_primitive::<Float32Type>(), 10)
+            .unwrap();
+        let results = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            results.num_rows(),
+            10,
+            "expected the requested k=10 nearest-neighbor results"
+        );
+    }
+
     /// Regression: a second `OptimizeOptions::append()` call on a steady-state
     /// vector index used to fall through to `optimize_vector_indices` and write
     /// a new UUID directory + manifest even though nothing had changed. The
@@ -1213,7 +1444,23 @@ mod tests {
         assert_eq!(results.num_rows(), 2);
         let mut id_arr = results["id"].as_primitive::<UInt32Type>().values().to_vec();
         id_arr.sort();
-        assert_eq!(id_arr, vec![0, 1000]);
+        // For exact indexes (e.g. IvfFlat) the top-2 nearest neighbors of the
+        // query vector are deterministic (id 0 in the first delta and id 1000
+        // in the second delta, since the same vector is duplicated across
+        // the two fragments). For approximate indexes (e.g. IvfPq, IvfHnswSq)
+        // the returned ids are not guaranteed to be exactly [0, 1000]; the key
+        // property this test verifies is that both delta indices are queried
+        // (i.e. one result comes from each delta), so we only assert that.
+        let is_approximate = !matches!(index_params.index_type(), IndexType::IvfFlat);
+        if is_approximate {
+            assert!(
+                id_arr[0] < TOTAL as u32 && id_arr[1] >= TOTAL as u32,
+                "expected one result from each delta index, got {:?}",
+                id_arr
+            );
+        } else {
+            assert_eq!(id_arr, vec![0, 1000]);
+        }
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs
index 702a8c4e49f..0f7a1d51471 100644
--- a/rust/lance/src/index/create.rs
+++ b/rust/lance/src/index/create.rs
@@ -12,8 +12,8 @@ use crate::{
         build_index_metadata_from_segments,
         scalar::{build_bitmap_index_segment, build_scalar_index},
         vector::{
-            LANCE_VECTOR_INDEX, VectorIndexParams, build_distributed_vector_index,
-            build_empty_vector_index, build_vector_index,
+            LANCE_VECTOR_INDEX, StageParams, VectorIndexParams, build_distributed_vector_index,
+            build_empty_vector_index, build_filtered_vector_index, build_vector_index,
         },
         vector_index_details, vector_index_details_default,
     },
@@ -24,7 +24,10 @@ use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
 use lance_index::{IndexParams, IndexType, scalar::CreatedIndex};
 use lance_index::{
     metrics::NoOpMetricsCollector,
-    scalar::{LANCE_SCALAR_INDEX, ScalarIndexParams, inverted::tokenizer::InvertedIndexParams},
+    scalar::{
+        LANCE_SCALAR_INDEX, ScalarIndexParams, index_files_to_table,
+        inverted::tokenizer::InvertedIndexParams, table_files_to_index,
+    },
 };
 use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
 use std::{collections::HashMap, future::IntoFuture, sync::Arc};
@@ -158,12 +161,16 @@ impl<'a> CreateIndexBuilder<'a> {
         let quoted_column: String = format_field_path(&names);
         let column = quoted_column.as_str();
 
-        // If train is true but dataset is empty, automatically set train to false
-        let train = if self.train {
-            self.dataset.count_rows(None).await? > 0
-        } else {
-            false
-        };
+        let vector_fragments_for_validation =
+            is_builtin_vector_index(self.index_type, self.params)
+                .then_some(self.fragments.as_deref())
+                .flatten();
+        let train = should_train_index(
+            self.dataset,
+            self.train,
+            vector_fragments_for_validation,
+        )
+        .await?;
 
         // Load indices from the disk.
         let indices = self.dataset.load_indices().await?;
@@ -367,24 +374,40 @@ impl<'a> CreateIndexBuilder<'a> {
                     })?;
                 let index_version = vec_params.index_type().version() as u32;
 
+                let effective_fragments =
+                    effective_vector_fragments(self.dataset, self.fragments.as_deref());
                 let files = if train {
                     // Check if this is distributed indexing (fragment-level)
-                    if let Some(fragments) = &self.fragments {
-                        // For distributed indexing, build only on specified fragments
-                        // This creates temporary index metadata without committing
-                        let (segment_uuid, files) = Box::pin(build_distributed_vector_index(
-                            self.dataset,
-                            column,
-                            &index_name,
-                            index_id,
-                            vec_params,
-                            fri,
-                            fragments,
-                            self.progress.clone(),
-                        ))
-                        .await?;
-                        output_index_uuid = segment_uuid;
-                        files
+                    if let Some(fragments) = effective_fragments.as_deref() {
+                        if vector_params_have_precomputed_ivf(vec_params) {
+                            // For distributed indexing, build only on specified fragments
+                            // This creates temporary index metadata without committing
+                            let (segment_uuid, files) = Box::pin(build_distributed_vector_index(
+                                self.dataset,
+                                column,
+                                &index_name,
+                                index_id,
+                                vec_params,
+                                fri,
+                                fragments,
+                                self.progress.clone(),
+                            ))
+                            .await?;
+                            output_index_uuid = segment_uuid;
+                            files
+                        } else {
+                            Box::pin(build_filtered_vector_index(
+                                self.dataset,
+                                column,
+                                &index_name,
+                                index_id,
+                                vec_params,
+                                fri,
+                                fragments,
+                                self.progress.clone(),
+                            ))
+                            .await?
+                        }
                     } else {
                         // Standard full dataset indexing
                         Box::pin(build_vector_index(
@@ -412,7 +435,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 CreatedIndex {
                     index_details: vector_index_details(vec_params),
                     index_version,
-                    files,
+                    files: table_files_to_index(files),
                 }
             }
             // Can't use if let Some(...) here because it's not stable yet.
@@ -451,7 +474,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 CreatedIndex {
                     index_details: vector_index_details_default(),
                     index_version: self.index_type.version() as u32,
-                    files,
+                    files: table_files_to_index(files),
                 }
             }
             (IndexType::FragmentReuse, _) => {
@@ -484,7 +507,7 @@ impl<'a> CreateIndexBuilder<'a> {
             index_version: created_index.index_version as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
-            files: Some(created_index.files),
+            files: Some(index_files_to_table(created_index.files)),
         })
         }
         .boxed()
@@ -651,7 +674,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 index_version: created_index.index_version as i32,
                 created_at: Some(chrono::Utc::now()),
                 base_id: None,
-                files: Some(created_index.files),
+                files: Some(index_files_to_table(created_index.files)),
             };
             let segments = vec![metadata.into_index_segment()?];
             let new_indices =
@@ -720,7 +743,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 index_version: created_index.index_version as i32,
                 created_at: Some(chrono::Utc::now()),
                 base_id: None,
-                files: Some(created_index.files),
+                files: Some(index_files_to_table(created_index.files)),
             });
         }
 
@@ -775,6 +798,56 @@ fn is_btree_scalar_params(params: &dyn IndexParams) -> bool {
         .is_some_and(|p| p.index_type.eq_ignore_ascii_case("btree"))
 }
 
+fn is_builtin_vector_index(index_type: IndexType, params: &dyn IndexParams) -> bool {
+    params.index_name() == LANCE_VECTOR_INDEX
+        && matches!(
+            index_type,
+            IndexType::Vector
+                | IndexType::IvfPq
+                | IndexType::IvfSq
+                | IndexType::IvfFlat
+                | IndexType::IvfRq
+                | IndexType::IvfHnswFlat
+                | IndexType::IvfHnswPq
+                | IndexType::IvfHnswSq
+        )
+        && params.as_any().is::<VectorIndexParams>()
+}
+
+async fn should_train_index(
+    dataset: &Dataset,
+    train: bool,
+    vector_fragments: Option<&[u32]>,
+) -> Result<bool> {
+    if !train {
+        return Ok(false);
+    }
+
+    if dataset.fragment_bitmap.is_empty() {
+        return Ok(false);
+    }
+
+    if let Some(fragment_ids) = vector_fragments {
+        dataset.get_fragments_from_ids(fragment_ids)?;
+        return Ok(true);
+    }
+
+    Ok(dataset.count_rows(None).await? > 0)
+}
+
+fn vector_params_have_precomputed_ivf(params: &VectorIndexParams) -> bool {
+    matches!(
+        params.stages.first(),
+        Some(StageParams::Ivf(ivf_params)) if ivf_params.centroids.is_some()
+    )
+}
+
+fn effective_vector_fragments(dataset: &Dataset, fragments: Option<&[u32]>) -> Option<Vec<u32>> {
+    let fragments = Dataset::normalize_fragment_ids(fragments?);
+    let fragment_bitmap: roaring::RoaringBitmap = fragments.iter().copied().collect();
+    (fragment_bitmap != *dataset.fragment_bitmap).then_some(fragments)
+}
+
 /// Validate that a user-supplied `index_uuid` is permitted for this build.
 fn ensure_index_uuid_allowed(
     index_type: IndexType,
@@ -782,20 +855,30 @@ fn ensure_index_uuid_allowed(
     fragments: Option<&Vec<u32>>,
     index_uuid: Option<&Uuid>,
 ) -> Result<()> {
-    let is_btree = index_type == IndexType::BTree
-        || params
-            .as_any()
-            .downcast_ref::<ScalarIndexParams>()
-            .map(|params| params.index_type.eq_ignore_ascii_case("btree"))
-            .unwrap_or(false);
-
-    if index_uuid.is_some() && fragments.is_some_and(|fragments| !fragments.is_empty()) && is_btree
+    let scalar_index_type = params
+        .as_any()
+        .downcast_ref::<ScalarIndexParams>()
+        .map(|params| params.index_type.as_str());
+    let unsafe_distributed_type = if index_type == IndexType::BTree
+        || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("btree"))
+    {
+        Some("BTree")
+    } else if index_type == IndexType::RTree
+        || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("rtree"))
+    {
+        Some("RTree")
+    } else {
+        None
+    };
+
+    if index_uuid.is_some()
+        && fragments.is_some_and(|fragments| !fragments.is_empty())
+        && let Some(index_type) = unsafe_distributed_type
     {
-        return Err(Error::invalid_input(
-            "index_uuid is no longer accepted for BTree distributed index builds; segment UUIDs \
-             are generated by Lance and returned in the index metadata."
-                .to_string(),
-        ));
+        return Err(Error::invalid_input(format!(
+            "index_uuid is no longer accepted for {index_type} distributed index builds; \
+                 segment UUIDs are generated by Lance and returned in the index metadata."
+        )));
     }
 
     Ok(())
@@ -859,12 +942,14 @@ mod tests {
     use lance_index::optimize::OptimizeOptions;
     use lance_index::progress::IndexBuildProgress;
     use lance_index::scalar::{
-        FullTextSearchQuery, SargableQuery, SearchResult, inverted::tokenizer::InvertedIndexParams,
+        BloomFilterQuery, FullTextSearchQuery, SargableQuery, SearchResult,
+        inverted::tokenizer::InvertedIndexParams,
     };
     use lance_index::vector::hnsw::builder::HnswBuildParams;
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
     use lance_linalg::distance::{DistanceType, MetricType};
+    use roaring::RoaringBitmap;
     use std::{collections::BTreeSet, ops::Bound, sync::Arc};
     use uuid::Uuid;
 
@@ -1152,6 +1237,50 @@ mod tests {
         IvfBuildParams::try_with_centroids(4, centroids).unwrap()
     }
 
+    async fn write_vector_fragment_dataset(uri: &str) -> Dataset {
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_get_frags_from_ordered_ids_accepts_unsorted_duplicates() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let first = fragments[0].id() as u32;
+        let second = fragments[1].id() as u32;
+
+        let resolved = dataset.get_frags_from_ordered_ids(&[second, first, second, u32::MAX]);
+
+        assert_eq!(resolved.len(), 4);
+        assert_eq!(resolved[0].as_ref().unwrap().id() as u32, second);
+        assert_eq!(resolved[1].as_ref().unwrap().id() as u32, first);
+        assert_eq!(resolved[2].as_ref().unwrap().id() as u32, second);
+        assert!(resolved[3].is_none());
+    }
+
     #[tokio::test]
     async fn test_execute_uncommitted() {
         // Test the complete workflow that covers the user's specified code pattern:
@@ -1706,6 +1835,170 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_bloomfilter_distributed_segments_merge_and_query() {
+        #[derive(serde::Serialize)]
+        struct BloomParams {
+            number_of_items: u64,
+            probability: f64,
+        }
+
+        let mut dataset = gen_batch()
+            .col("value", lance_datagen::array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(8))
+            .await
+            .unwrap();
+
+        let params =
+            ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter)
+                .with_params(&BloomParams {
+                    number_of_items: 4,
+                    probability: 0.000_001,
+                });
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 3);
+        let mut staged = Vec::with_capacity(fragments.len());
+        for fragment in &fragments {
+            staged.push(
+                CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::BloomFilter, &params)
+                    .name("value_bloom_segments".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+
+        let mut future_segment = staged[1].clone();
+        future_segment.dataset_version = dataset.manifest.version + 1;
+        let error = dataset
+            .merge_existing_index_segments(vec![staged[0].clone(), future_segment])
+            .await
+            .unwrap_err();
+        assert!(
+            error.to_string().contains("future dataset version"),
+            "unexpected merge error: {error}"
+        );
+
+        dataset
+            .commit_existing_index_segments("value_bloom_segments", "value", staged.clone())
+            .await
+            .unwrap();
+        let logical = crate::index::scalar_logical::open_named_scalar_index(
+            &dataset,
+            "value",
+            "value_bloom_segments",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            logical.calculate_included_frags().await.unwrap(),
+            dataset.fragment_bitmap.as_ref().clone()
+        );
+
+        let mut trimmed = staged.clone();
+        trimmed[0].fragment_bitmap = Some(RoaringBitmap::new());
+        let trimmed_merged = dataset
+            .merge_existing_index_segments(trimmed)
+            .await
+            .unwrap();
+        assert_eq!(
+            trimmed_merged.fragment_bitmap.as_ref().unwrap(),
+            &RoaringBitmap::from_iter(fragments[1..].iter().map(|fragment| fragment.id() as u32))
+        );
+
+        let incompatible_params =
+            ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter)
+                .with_params(&BloomParams {
+                    number_of_items: 8,
+                    probability: 0.01,
+                });
+        let incompatible = CreateIndexBuilder::new(
+            &mut dataset,
+            &["value"],
+            IndexType::BloomFilter,
+            &incompatible_params,
+        )
+        .name("value_bloom_incompatible".to_string())
+        .fragments(vec![fragments[1].id() as u32])
+        .execute_uncommitted()
+        .await
+        .unwrap();
+        let err = dataset
+            .merge_existing_index_segments(vec![staged[0].clone(), incompatible])
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("different parameters: number_of_items=4"),
+            "unexpected merge error: {err}"
+        );
+
+        let expected_dataset_version = staged
+            .iter()
+            .map(|segment| segment.dataset_version)
+            .min()
+            .unwrap();
+        let merged = dataset.merge_existing_index_segments(staged).await.unwrap();
+        assert_eq!(merged.dataset_version, expected_dataset_version);
+        assert_eq!(
+            merged.fragment_bitmap.as_ref().unwrap(),
+            dataset.fragment_bitmap.as_ref()
+        );
+        assert!(
+            merged
+                .index_details
+                .as_ref()
+                .unwrap()
+                .type_url
+                .ends_with("BloomFilterIndexDetails")
+        );
+        dataset
+            .commit_existing_index_segments("value_bloom_merged", "value", vec![merged])
+            .await
+            .unwrap();
+        let committed = dataset
+            .load_indices_by_name("value_bloom_merged")
+            .await
+            .unwrap();
+        assert_eq!(committed[0].dataset_version, expected_dataset_version);
+        let merged_logical = crate::index::scalar_logical::open_named_scalar_index(
+            &dataset,
+            "value",
+            "value_bloom_merged",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        let candidates = merged_logical
+            .search(
+                &BloomFilterQuery::Equals(ScalarValue::Int32(Some(17))),
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        assert!(
+            candidates
+                .row_addrs()
+                .true_rows()
+                .row_addrs()
+                .unwrap()
+                .map(u64::from)
+                .any(|row_addr| row_addr == (2_u64 << 32) + 1)
+        );
+
+        let result = dataset
+            .scan()
+            .filter("value = 17")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(result.num_rows(), 1);
+        assert_eq!(result["value"].as_primitive::<Int32Type>().value(0), 17);
+    }
+
     #[tokio::test]
     async fn test_vector_execute_uncommitted_segments_commit_without_staging() {
         let tmpdir = TempStrDir::default();
@@ -1809,6 +2102,218 @@ mod tests {
         assert!(result.num_rows() > 0);
     }
 
+    #[tokio::test]
+    async fn test_vector_explicit_all_fragments_uses_full_build_without_precomputed_ivf() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragment_ids = dataset
+            .get_fragments()
+            .iter()
+            .map(|fragment| fragment.id() as u32)
+            .collect::<Vec<_>>();
+        assert!(fragment_ids.len() >= 2);
+
+        let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10);
+        params.version(crate::index::vector::IndexFileVersion::Legacy);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(fragment_ids)
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert_eq!(
+            segment.fragment_bitmap.as_ref().unwrap(),
+            dataset.fragment_bitmap.as_ref()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_precomputed_ivf_num_partitions_mismatch_errors() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await;
+        let centroid_count = ivf_params.centroids.as_ref().unwrap().len();
+        ivf_params.num_partitions = Some(centroid_count + 1);
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params);
+
+        let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+            .name("vector_idx".to_string())
+            .execute_uncommitted()
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.to_string().contains(&format!(
+                "num_partitions {} does not match precomputed IVF centroids length {}",
+                centroid_count + 1,
+                centroid_count
+            )),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_subset_legacy_ivf_pq_rejects_filtered_build() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10);
+        params.version(crate::index::vector::IndexFileVersion::Legacy);
+
+        let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+            .name("vector_idx".to_string())
+            .fragments(vec![fragments[0].id() as u32])
+            .execute_uncommitted()
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.to_string()
+                .contains("filtered IVF_PQ builds do not support legacy format"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_subset_fragments_train_without_precomputed_ivf() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 3);
+        let selected = vec![
+            fragments[1].id() as u32,
+            fragments[0].id() as u32,
+            fragments[1].id() as u32,
+        ];
+        let expected_bitmap = selected.iter().copied().collect::<roaring::RoaringBitmap>();
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(selected)
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert_eq!(segment.fragment_bitmap.as_ref().unwrap(), &expected_bitmap);
+    }
+
+    #[tokio::test]
+    async fn test_vector_merge_rejects_independently_trained_fragment_segments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let mut segments = Vec::new();
+        for fragment in fragments.iter().take(2) {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            segments.push(segment);
+        }
+
+        let err = dataset
+            .merge_existing_index_segments(segments)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("IVF centroids mismatch across shards"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_optimize_rejects_independently_trained_fragment_segments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let mut segments = Vec::new();
+        for fragment in fragments.iter().take(2) {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            segments.push(segment);
+        }
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", segments)
+            .await
+            .unwrap();
+
+        let err = dataset
+            .optimize_indices(&OptimizeOptions::merge(2))
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("vector index segments do not share IVF centroids"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_empty_fragments_with_precomputed_ivf_builds_empty_segment() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await;
+        let expected_partitions = ivf_params.centroids.as_ref().unwrap().len();
+        ivf_params.num_partitions = None;
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(vec![])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert!(segment.fragment_bitmap.as_ref().unwrap().is_empty());
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![segment])
+            .await
+            .unwrap();
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+        let metadata = logical_index.metadatas().next().unwrap();
+        assert_eq!(
+            logical_index.as_ivf().unwrap().num_partitions_per_segment(),
+            vec![(metadata.uuid, expected_partitions)]
+        );
+    }
+
     #[tokio::test]
     async fn test_commit_existing_index_segments_vector_commits_multi_segment_logical_index() {
         let tmpdir = TempStrDir::default();
@@ -2566,7 +3071,7 @@ mod tests {
         let mut legacy_segment = segment.clone();
         legacy_segment.uuid = legacy_uuid;
         legacy_segment.index_version = LABEL_LIST_NULLS_MIN_VERSION;
-        legacy_segment.files = Some(vec![legacy_file]);
+        legacy_segment.files = Some(index_files_to_table(vec![legacy_file]));
 
         let err = dataset
             .merge_existing_index_segments(vec![legacy_segment])
@@ -2626,8 +3131,10 @@ mod tests {
                 vec![IndexSegment::new(
                     uuid,
                     dataset.fragment_bitmap.as_ref().clone(),
+                    [dataset.schema().field("vector").unwrap().id],
                     Arc::new(vector_index_details(&params)),
                     IndexType::IvfHnswFlat.version(),
+                    dataset.manifest.version,
                 )],
             )
             .await
diff --git a/rust/lance/src/index/mem_wal.rs b/rust/lance/src/index/mem_wal.rs
index de2c70b62d2..699338085b4 100644
--- a/rust/lance/src/index/mem_wal.rs
+++ b/rust/lance/src/index/mem_wal.rs
@@ -5,18 +5,18 @@
 //!
 //! The MemWAL Index stores:
 //! - Configuration (sharding_specs, maintained_indexes)
-//! - Merge progress (merged_generations per shard)
+//! - SSTable compaction progress
 //! - Shard state snapshots (eventually consistent)
 //!
 //! Writers no longer update the index on every write. Instead, they update
 //! shard manifests directly. This module provides functions to:
 //! - Load the MemWAL index
-//! - Update merged generations (called during merge-insert commits)
+//! - Update compacted SSTables (called during merge-insert commits)
 
 use std::sync::Arc;
 
 use lance_core::{Error, Result};
-use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexDetails, MergedGeneration};
+use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexDetails};
 use lance_table::format::{IndexMetadata, pb};
 use uuid::Uuid;
 
@@ -45,15 +45,16 @@ pub(crate) fn open_mem_wal_index(index: IndexMetadata) -> Result<Arc<MemWalIndex
     )?)))
 }
 
-/// Update merged_generations in the MemWAL index.
+/// Update `compacted_sstables` in the MemWAL index.
+///
 /// This is called during merge-insert commits to atomically record which
-/// generations have been merged to the base table.
-pub(crate) fn update_mem_wal_index_merged_generations(
+/// SSTables have been compacted into the base table.
+pub(crate) fn update_mem_wal_index_compacted_sstables(
     indices: &mut Vec<IndexMetadata>,
     dataset_version: u64,
-    new_merged_generations: Vec<MergedGeneration>,
+    new_compacted_sstables: Vec<CompactedSsTable>,
 ) -> Result<()> {
-    if new_merged_generations.is_empty() {
+    if new_compacted_sstables.is_empty() {
         return Ok(());
     }
 
@@ -65,26 +66,26 @@ pub(crate) fn update_mem_wal_index_merged_generations(
         let current_meta = indices.remove(pos);
         let mut details = load_mem_wal_index_details(current_meta)?;
 
-        // Update merged_generations - for each shard, keep the higher generation
-        for new_mg in new_merged_generations {
+        // Update compacted_sstables - for each shard, keep the higher generation
+        for new_sstable in new_compacted_sstables {
             if let Some(existing) = details
-                .merged_generations
+                .compacted_sstables
                 .iter_mut()
-                .find(|mg| mg.shard_id == new_mg.shard_id)
+                .find(|sstable| sstable.shard_id == new_sstable.shard_id)
             {
-                if new_mg.generation > existing.generation {
-                    existing.generation = new_mg.generation;
+                if new_sstable.generation > existing.generation {
+                    existing.generation = new_sstable.generation;
                 }
             } else {
-                details.merged_generations.push(new_mg);
+                details.compacted_sstables.push(new_sstable);
             }
         }
 
         new_mem_wal_index_meta(dataset_version, details)?
     } else {
-        // Create new MemWAL index with just the merged generations
+        // Create a MemWAL index containing only compaction progress.
         let details = MemWalIndexDetails {
-            merged_generations: new_merged_generations,
+            compacted_sstables: new_compacted_sstables,
             ..Default::default()
         };
         new_mem_wal_index_meta(dataset_version, details)?
@@ -163,7 +164,7 @@ mod tests {
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -177,7 +178,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -200,7 +201,7 @@ mod tests {
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -213,7 +214,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -237,7 +238,7 @@ mod tests {
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -251,7 +252,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -275,7 +276,7 @@ mod tests {
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard1, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard1, 10)],
             },
             None,
         );
@@ -289,7 +290,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard2, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard2, 5)],
             },
             None,
         );
@@ -312,11 +313,11 @@ mod tests {
             .unwrap()
             .clone();
         let details = load_mem_wal_index_details(mem_wal_idx).unwrap();
-        assert_eq!(details.merged_generations.len(), 2);
+        assert_eq!(details.compacted_sstables.len(), 2);
     }
 
     /// Test that CreateIndex of MemWalIndex can be rebased against UpdateMemWalState.
-    /// The merged_generations from UpdateMemWalState should be merged into CreateIndex.
+    /// The compacted_sstables from UpdateMemWalState should be included in CreateIndex.
     #[tokio::test]
     async fn test_create_index_rebase_against_update_mem_wal_state() {
         let dataset = test_dataset().await;
@@ -326,7 +327,7 @@ mod tests {
         let txn1 = Transaction::new(
             dataset.manifest.version,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -336,7 +337,7 @@ mod tests {
             .unwrap();
 
         // CreateIndex of MemWalIndex based on old version (before UpdateMemWalState)
-        // This should succeed and merge the generations
+        // This should succeed and combine the compaction progress.
         let details = MemWalIndexDetails {
             num_shards: 1,
             ..Default::default()
@@ -359,7 +360,7 @@ mod tests {
             result
         );
 
-        // Verify the merged_generations from UpdateMemWalState were merged into CreateIndex
+        // Verify the compacted_sstables from UpdateMemWalState were included in CreateIndex
         let dataset = result.unwrap();
         let mem_wal_idx = dataset
             .load_indices()
@@ -370,9 +371,9 @@ mod tests {
             .unwrap()
             .clone();
         let details = load_mem_wal_index_details(mem_wal_idx).unwrap();
-        assert_eq!(details.merged_generations.len(), 1);
-        assert_eq!(details.merged_generations[0].shard_id, shard);
-        assert_eq!(details.merged_generations[0].generation, 10);
+        assert_eq!(details.compacted_sstables.len(), 1);
+        assert_eq!(details.compacted_sstables[0].shard_id, shard);
+        assert_eq!(details.compacted_sstables[0].generation, 10);
         assert_eq!(details.num_shards, 1); // Config from CreateIndex preserved
     }
 
@@ -382,9 +383,9 @@ mod tests {
         let dataset = test_dataset().await;
         let shard = Uuid::new_v4();
 
-        // First commit CreateIndex of MemWalIndex with merged_generations
+        // First commit CreateIndex of MemWalIndex with compacted_sstables
         let details = MemWalIndexDetails {
-            merged_generations: vec![MergedGeneration::new(shard, 10)],
+            compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             ..Default::default()
         };
         let mem_wal_index = new_mem_wal_index_meta(dataset.manifest.version, details).unwrap();
@@ -406,7 +407,7 @@ mod tests {
         let txn2 = Transaction::new(
             dataset.manifest.version - 1, // Based on old version
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -420,74 +421,147 @@ mod tests {
     }
 
     #[test]
-    fn test_update_merged_generations() {
+    fn test_update_compacted_sstables() {
         let mut indices = Vec::new();
         let shard1 = Uuid::new_v4();
         let shard2 = Uuid::new_v4();
 
         // First update - creates new index
-        update_mem_wal_index_merged_generations(
+        update_mem_wal_index_compacted_sstables(
             &mut indices,
             1,
-            vec![MergedGeneration::new(shard1, 5)],
+            vec![CompactedSsTable::new(shard1, 5)],
         )
         .unwrap();
 
         assert_eq!(indices.len(), 1);
         let details = load_mem_wal_index_details(indices[0].clone()).unwrap();
-        assert_eq!(details.merged_generations.len(), 1);
-        assert_eq!(details.merged_generations[0].shard_id, shard1);
-        assert_eq!(details.merged_generations[0].generation, 5);
+        assert_eq!(details.compacted_sstables.len(), 1);
+        assert_eq!(details.compacted_sstables[0].shard_id, shard1);
+        assert_eq!(details.compacted_sstables[0].generation, 5);
 
         // Second update - updates existing shard
-        update_mem_wal_index_merged_generations(
+        update_mem_wal_index_compacted_sstables(
             &mut indices,
             2,
-            vec![MergedGeneration::new(shard1, 10)],
+            vec![CompactedSsTable::new(shard1, 10)],
         )
         .unwrap();
 
         assert_eq!(indices.len(), 1);
         let details = load_mem_wal_index_details(indices[0].clone()).unwrap();
-        assert_eq!(details.merged_generations.len(), 1);
-        assert_eq!(details.merged_generations[0].generation, 10);
+        assert_eq!(details.compacted_sstables.len(), 1);
+        assert_eq!(details.compacted_sstables[0].generation, 10);
 
         // Third update - adds new shard
-        update_mem_wal_index_merged_generations(
+        update_mem_wal_index_compacted_sstables(
             &mut indices,
             3,
-            vec![MergedGeneration::new(shard2, 3)],
+            vec![CompactedSsTable::new(shard2, 3)],
         )
         .unwrap();
 
         assert_eq!(indices.len(), 1);
         let details = load_mem_wal_index_details(indices[0].clone()).unwrap();
-        assert_eq!(details.merged_generations.len(), 2);
+        assert_eq!(details.compacted_sstables.len(), 2);
 
         // Fourth update - lower generation should not update
-        update_mem_wal_index_merged_generations(
+        update_mem_wal_index_compacted_sstables(
             &mut indices,
             4,
-            vec![MergedGeneration::new(shard1, 8)], // lower than 10
+            vec![CompactedSsTable::new(shard1, 8)], // lower than 10
         )
         .unwrap();
 
         let details = load_mem_wal_index_details(indices[0].clone()).unwrap();
-        let r1_mg = details
-            .merged_generations
+        let shard1_sstable = details
+            .compacted_sstables
             .iter()
-            .find(|mg| mg.shard_id == shard1)
+            .find(|sstable| sstable.shard_id == shard1)
             .unwrap();
-        assert_eq!(r1_mg.generation, 10); // Should still be 10
+        assert_eq!(shard1_sstable.generation, 10); // Should still be 10
     }
 
     #[test]
-    fn test_empty_merged_generations_noop() {
+    fn test_empty_compacted_sstables_noop() {
         let mut indices = Vec::new();
 
         // Empty update should be a no-op
-        update_mem_wal_index_merged_generations(&mut indices, 1, vec![]).unwrap();
+        update_mem_wal_index_compacted_sstables(&mut indices, 1, vec![]).unwrap();
 
         assert!(indices.is_empty());
     }
+
+    /// Regression: a committed `__mem_wal` (legitimately `fragment_bitmap:
+    /// None`) must not break `describe_indices` — the path behind lancedb's
+    /// `list_indices`/`wait_for_index`. It's described as zero indexed rows,
+    /// like `__frag_reuse`.
+    #[tokio::test]
+    async fn test_describe_indices_includes_mem_wal_system_index() {
+        use crate::index::DatasetIndexExt;
+        use lance_index::IndexType;
+        use lance_index::scalar::ScalarIndexParams;
+
+        let mut dataset = test_dataset().await;
+
+        // A real user index that describe_indices must keep returning.
+        dataset
+            .create_index(
+                &["a"],
+                IndexType::Scalar,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Commit a __mem_wal index, as WAL provisioning does in production.
+        let shard = Uuid::new_v4();
+        let txn = Transaction::new(
+            dataset.manifest.version,
+            Operation::UpdateMemWalState {
+                compacted_sstables: vec![CompactedSsTable::new(shard, 1)],
+            },
+            None,
+        );
+        let dataset = CommitBuilder::new(Arc::new(dataset))
+            .execute(txn)
+            .await
+            .unwrap();
+
+        // The system index is present with no fragment_bitmap (by design).
+        let mem_wal = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .find(|i| i.name == MEM_WAL_INDEX_NAME)
+            .unwrap()
+            .clone();
+        assert!(mem_wal.fragment_bitmap.is_none());
+
+        // describe_indices describes the bitmap-less __mem_wal alongside the
+        // real index instead of erroring.
+        let descriptions = dataset.describe_indices(None).await.unwrap();
+        let mem_wal_desc = descriptions
+            .iter()
+            .find(|d| d.name() == MEM_WAL_INDEX_NAME)
+            .expect("__mem_wal must be described, not skipped");
+        assert_eq!(
+            mem_wal_desc.index_type(),
+            "MemWal",
+            "system index type must resolve via infer_system_index_type"
+        );
+        assert_eq!(
+            mem_wal_desc.rows_indexed(),
+            0,
+            "a bitmap-less system index indexes zero rows"
+        );
+        assert_eq!(
+            descriptions.len(),
+            2,
+            "both the real scalar index and __mem_wal must be listed"
+        );
+    }
 }
diff --git a/rust/lance/src/index/prefilter.rs b/rust/lance/src/index/prefilter.rs
index 071f1b8893d..1e02b6f807a 100644
--- a/rust/lance/src/index/prefilter.rs
+++ b/rust/lance/src/index/prefilter.rs
@@ -52,6 +52,10 @@ pub struct DatasetPreFilter {
     // Fragment IDs whose data is still in the index but has been removed from the dataset.
     // Used by FTS merge-on-read to prune stale fragments at search time.
     pub(super) deleted_fragments: Option<RoaringBitmap>,
+    // Row addresses whose index entries are stale due to a newer data overlay committed after
+    // the index was built. Computed synchronously at plan time and ANDead into the final mask
+    // so the index never returns those rows.
+    pub(super) overlay_block: Option<RowAddrMask>,
     // When the tasks are finished this is the combined filter
     pub(super) final_mask: Mutex<OnceCell<Arc<RowAddrMask>>>,
 }
@@ -83,6 +87,7 @@ impl DatasetPreFilter {
             deleted_ids,
             filtered_ids,
             deleted_fragments: None,
+            overlay_block: None,
             final_mask: Mutex::new(OnceCell::new()),
         }
     }
@@ -226,6 +231,13 @@ impl DatasetPreFilter {
         self.deleted_fragments = Some(fragments);
     }
 
+    /// Block specific row addresses from index results because their index entries are stale
+    /// due to a data overlay committed after the index was built.
+    pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self {
+        self.overlay_block = Some(block);
+        self
+    }
+
     /// Creates a task to load a mask that filters out deleted rows and,
     /// when `restrict_to_fragments` is true, also restricts results to only
     /// the given `fragments`.
@@ -383,6 +395,9 @@ impl PreFilter for DatasetPreFilter {
                 }
                 combined = combined & RowAddrMask::from_block(block_list);
             }
+            if let Some(overlay_block) = &self.overlay_block {
+                combined = combined & overlay_block.clone();
+            }
             Arc::new(combined)
         });
 
@@ -393,6 +408,7 @@ impl PreFilter for DatasetPreFilter {
         self.deleted_ids.is_none()
             && self.filtered_ids.is_none()
             && self.deleted_fragments.is_none()
+            && self.overlay_block.is_none()
     }
 
     /// Get the row id mask for this prefilter
diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs
index b2ee8e0426a..ae4d61058b8 100644
--- a/rust/lance/src/index/scalar.rs
+++ b/rust/lance/src/index/scalar.rs
@@ -5,13 +5,16 @@
 //!
 
 pub(crate) mod bitmap;
+pub(crate) mod bloomfilter;
 pub(crate) mod btree;
 pub(crate) mod fmindex;
 pub(crate) mod inverted;
 pub(crate) mod label_list;
+#[cfg(feature = "geo")]
+pub(crate) mod rtree;
 pub(crate) mod zonemap;
 
-pub use inverted::{load_segment_details, load_segments};
+pub use inverted::{load_segment_details, load_segment_params, load_segments};
 
 pub use crate::index::scalar_logical::{LogicalScalarIndex, load_named_scalar_segments};
 
@@ -35,6 +38,7 @@ use lance_core::datatypes::Field;
 use lance_core::utils::tracing::{IO_TYPE_OPEN_SCALAR, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ADDR, ROW_ID, Result};
 use lance_datafusion::exec::LanceExecutionOptions;
+use lance_index::frag_reuse::FragReuseIndexHandle;
 use lance_index::metrics::{MetricsCollector, NoOpMetricsCollector};
 use lance_index::pbold::{
     BTreeIndexDetails, BitmapIndexDetails, InvertedIndexDetails, LabelListIndexDetails,
@@ -460,7 +464,7 @@ pub async fn open_scalar_index(
         .for_index(&index.uuid, frag_reuse_index.as_ref().map(|f| &f.uuid));
 
     let frag_reuse_index: Option<Arc<dyn RowIdRemapper>> =
-        frag_reuse_index.map(|f| f as Arc<dyn RowIdRemapper>);
+        frag_reuse_index.map(|f| Arc::new(FragReuseIndexHandle(f)) as Arc<dyn RowIdRemapper>);
 
     // Runs only on a cold miss, and at most once even under concurrent opens
     // (the plugin coalesces). The compat check lives here because a warm hit was
@@ -2304,4 +2308,70 @@ mod tests {
             "Should have 0 rows with value='banana' after deletion"
         );
     }
+
+    // End-to-end: create index → delete a whole fragment → search (via index) →
+    // update index → search again.  No deleted rows should ever appear.
+    #[tokio::test]
+    async fn test_zonemap_search_with_deleted_fragment_before_and_after_update() {
+        use arrow::datatypes::Int32Type;
+        use lance_datagen::array;
+        use lance_index::IndexType;
+        use lance_index::optimize::OptimizeOptions;
+        use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+        // 3 fragments × 10 rows: id 0-9 (frag 0), 10-19 (frag 1), 20-29 (frag 2).
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(10))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        ds.create_index(&["id"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Delete the middle fragment entirely.
+        ds.delete("id >= 10 AND id < 20").await.unwrap();
+
+        // Helper: run a filter scan and return the sorted id values.
+        async fn live_ids(ds: &crate::Dataset) -> Vec<i32> {
+            let batch = ds
+                .scan()
+                .filter("id >= 0")
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let mut ids: Vec<i32> = batch["id"]
+                .as_any()
+                .downcast_ref::<arrow_array::Int32Array>()
+                .unwrap()
+                .values()
+                .to_vec();
+            ids.sort_unstable();
+            ids
+        }
+
+        // --- Before index update ---
+        let ids = live_ids(&ds).await;
+        assert_eq!(ids.len(), 20, "expected 20 live rows before index update");
+        assert!(
+            ids.iter().all(|&id| !(10..20).contains(&id)),
+            "deleted fragment rows (id 10-19) must not appear before index update; got: {:?}",
+            ids
+        );
+
+        // Update the zone map index to reflect the deletion.
+        ds.optimize_indices(&OptimizeOptions::new()).await.unwrap();
+
+        // --- After index update ---
+        let ids = live_ids(&ds).await;
+        assert_eq!(ids.len(), 20, "expected 20 live rows after index update");
+        assert!(
+            ids.iter().all(|&id| !(10..20).contains(&id)),
+            "deleted fragment rows (id 10-19) must not appear after index update; got: {:?}",
+            ids
+        );
+    }
 }
diff --git a/rust/lance/src/index/scalar/bitmap.rs b/rust/lance/src/index/scalar/bitmap.rs
index 2eb5702ee28..84c8b6a8b91 100644
--- a/rust/lance/src/index/scalar/bitmap.rs
+++ b/rust/lance/src/index/scalar/bitmap.rs
@@ -3,6 +3,7 @@
 
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::scalar::bitmap::BitmapIndex;
+use lance_index::scalar::index_files_to_table;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_table::format::IndexMetadata;
 use roaring::RoaringBitmap;
@@ -70,7 +71,7 @@ pub(in crate::index) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
diff --git a/rust/lance/src/index/scalar/bloomfilter.rs b/rust/lance/src/index/scalar/bloomfilter.rs
new file mode 100644
index 00000000000..6bd002ae88a
--- /dev/null
+++ b/rust/lance/src/index/scalar/bloomfilter.rs
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::bloomfilter::{BloomFilterIndex, MAX_BINARY_VALUE_BUFFER_LEN};
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_table::format::IndexMetadata;
+use roaring::RoaringBitmap;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+/// Merge one caller-defined group of source BloomFilter segments into a single segment.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+    let dataset_version = segments
+        .iter()
+        .map(|segment| segment.dataset_version)
+        .min()
+        .unwrap_or(dataset.manifest.version);
+    if segments.iter().all(|segment| segment.files.is_some()) {
+        let total_size = segments
+            .iter()
+            .filter_map(IndexMetadata::total_size_bytes)
+            .try_fold(0u64, u64::checked_add)
+            .ok_or_else(|| Error::invalid_input("BloomFilter segment sizes overflowed u64"))?;
+        if total_size > MAX_BINARY_VALUE_BUFFER_LEN {
+            return Err(Error::invalid_input(format!(
+                "BloomFilter segment files total {total_size} bytes, exceeding the Arrow \
+                 BinaryArray merge limit of {MAX_BINARY_VALUE_BUFFER_LEN} bytes; merge fewer \
+                 segments at a time"
+            )));
+        }
+    }
+
+    let mut fragment_bitmap = RoaringBitmap::new();
+    let dataset_fragments = dataset.fragment_bitmap.as_ref();
+    let fragment_filters = segments
+        .iter()
+        .map(|segment| {
+            segment
+                .effective_fragment_bitmap(dataset_fragments)
+                .ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "CreateIndex: segment {} is missing fragment coverage",
+                        segment.uuid
+                    ))
+                })
+        })
+        .collect::<Result<Vec<_>>>()?;
+    for effective in &fragment_filters {
+        fragment_bitmap |= effective;
+    }
+
+    let mut scalar_indices = Vec::with_capacity(segments.len());
+    for (position, (segment, effective)) in segments.iter().zip(&fragment_filters).enumerate() {
+        if effective.is_empty() && !(fragment_bitmap.is_empty() && position == 0) {
+            continue;
+        }
+        let scalar_index =
+            super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?;
+        scalar_indices.push((segment.uuid, scalar_index, effective));
+    }
+
+    let mut source_indices = Vec::with_capacity(scalar_indices.len());
+    for (segment_uuid, scalar_index, fragment_filter) in &scalar_indices {
+        let bloomfilter_index = scalar_index
+            .as_any()
+            .downcast_ref::<BloomFilterIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "merge_existing_index_segments: expected bloom filter segment {}, got {:?}",
+                    segment_uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push((bloomfilter_index, *fragment_filter));
+    }
+
+    let new_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
+    let created_index =
+        lance_index::scalar::bloomfilter::merge_bloomfilter_indices(&source_indices, &new_store)
+            .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar/btree.rs b/rust/lance/src/index/scalar/btree.rs
index 4339b8c183b..7089997721c 100644
--- a/rust/lance/src/index/scalar/btree.rs
+++ b/rust/lance/src/index/scalar/btree.rs
@@ -15,7 +15,7 @@ use lance_index::pbold::BTreeIndexDetails;
 use lance_index::scalar::btree::BTreeIndex;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
-use lance_index::scalar::{CreatedIndex, OldIndexDataFilter};
+use lance_index::scalar::{CreatedIndex, OldIndexDataFilter, index_files_to_table};
 use lance_table::format::IndexMetadata;
 use uuid::Uuid;
 
@@ -161,6 +161,6 @@ pub(crate) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
     })
 }
diff --git a/rust/lance/src/index/scalar/fmindex.rs b/rust/lance/src/index/scalar/fmindex.rs
index 32684ebf9ab..9c1baadbbb6 100644
--- a/rust/lance/src/index/scalar/fmindex.rs
+++ b/rust/lance/src/index/scalar/fmindex.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_index::scalar::index_files_to_table;
 use lance_table::format::IndexMetadata;
 use roaring::RoaringBitmap;
 use std::sync::Arc;
@@ -76,7 +77,7 @@ pub(in crate::index) async fn merge_segments(
             index_version: created_index.index_version as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
-            files: Some(created_index.files),
+            files: Some(index_files_to_table(created_index.files)),
             ..segments[0].clone()
         });
     }
@@ -111,7 +112,7 @@ pub(in crate::index) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index 000d2c3139c..e49daa67b5e 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -11,7 +11,8 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use lance_core::ROW_ID;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::pbold::InvertedIndexDetails;
-use lance_index::scalar::inverted::InvertedIndex;
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexParams};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
 use lance_table::format::IndexMetadata;
@@ -137,7 +138,7 @@ pub(crate) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
@@ -182,9 +183,11 @@ pub async fn load_segments(dataset: &Dataset, column: &str) -> Result<Option<Vec
 /// Load and validate the shared [`InvertedIndexDetails`] across committed
 /// segments returned by [`load_segments`].
 ///
-/// All segments are required to agree on their decoded `InvertedIndexDetails`
-/// payload (analyzer, tokenizer, position settings, etc.); inconsistent
-/// segments return an error. Returns the canonical details that may be used
+/// All segments are required to agree on their semantic `InvertedIndexDetails`
+/// payload (tokenizer, position settings, etc.); inconsistent
+/// segments return an error. Details are canonicalized before comparison so
+/// legacy segments that omit default fields remain compatible with newly
+/// written text FTS segments. Returns the canonical details that may be used
 /// when constructing a tokenizer or running a query against the index.
 pub async fn load_segment_details(
     dataset: &Dataset,
@@ -200,6 +203,7 @@ pub async fn load_segment_details(
                     "failed to decode InvertedIndexDetails payload: {err}"
                 ))
             })?;
+        let details = canonicalize_inverted_index_details(details)?;
         match &expected_details {
             Some(expected) if expected != &details => {
                 return Err(Error::invalid_input(format!(
@@ -219,6 +223,22 @@ pub async fn load_segment_details(
     })
 }
 
+fn canonicalize_inverted_index_details(
+    details: InvertedIndexDetails,
+) -> Result<InvertedIndexDetails> {
+    let params = InvertedIndexParams::try_from(&details)?;
+    InvertedIndexDetails::try_from(&params)
+}
+
+/// Read one segment's [`InvertedIndexParams`]
+pub async fn load_segment_params(
+    dataset: &Dataset,
+    segment: &IndexMetadata,
+) -> Result<InvertedIndexParams> {
+    let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?;
+    InvertedIndex::load_params(&store).await
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -231,4 +251,16 @@ mod tests {
         let decoded = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap();
         assert_eq!(decoded, InvertedIndexDetails::default());
     }
+
+    #[test]
+    fn canonicalize_inverted_details_accepts_legacy_empty_details() {
+        let legacy = InvertedIndexDetails::default();
+        let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
+
+        assert_ne!(legacy, current);
+        assert_eq!(
+            canonicalize_inverted_index_details(legacy).unwrap(),
+            canonicalize_inverted_index_details(current).unwrap()
+        );
+    }
 }
diff --git a/rust/lance/src/index/scalar/label_list.rs b/rust/lance/src/index/scalar/label_list.rs
index 27bc49643bb..884346f0d6b 100644
--- a/rust/lance/src/index/scalar/label_list.rs
+++ b/rust/lance/src/index/scalar/label_list.rs
@@ -3,6 +3,7 @@
 
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::scalar::IndexStore;
+use lance_index::scalar::index_files_to_table;
 use lance_index::scalar::label_list::{
     BITMAP_LOOKUP_NAME, LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, LabelListIndex,
 };
@@ -142,7 +143,7 @@ pub(in crate::index) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
diff --git a/rust/lance/src/index/scalar/rtree.rs b/rust/lance/src/index/scalar/rtree.rs
new file mode 100644
index 00000000000..de9fdece137
--- /dev/null
+++ b/rust/lance/src/index/scalar/rtree.rs
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_index::scalar::rtree::RTreeIndex;
+use lance_index::scalar::{OldIndexDataFilter, index_files_to_table};
+use lance_select::RowSetOps;
+use lance_table::format::IndexMetadata;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+fn filter_keeps_nothing(filter: &Option<OldIndexDataFilter>) -> bool {
+    match filter {
+        Some(OldIndexDataFilter::Fragments { to_keep, .. }) => to_keep.is_empty(),
+        Some(OldIndexDataFilter::RowIds(valid)) => valid.is_empty(),
+        None => false,
+    }
+}
+
+/// Merge one caller-defined group of source RTree segments into a single segment.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+    let dataset_version = segments
+        .iter()
+        .map(|segment| segment.dataset_version)
+        .min()
+        .unwrap_or(dataset.manifest.version);
+    let segment_refs = segments.iter().collect::<Vec<_>>();
+    let (fragment_bitmap, old_data_filters) =
+        crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?;
+
+    let mut source_indices = Vec::with_capacity(segments.len());
+    let mut source_filters = Vec::with_capacity(old_data_filters.len());
+    let all_keep_nothing = old_data_filters.iter().all(filter_keeps_nothing);
+    for (position, (segment, filter)) in segments.iter().zip(&old_data_filters).enumerate() {
+        if filter_keeps_nothing(filter) && !(all_keep_nothing && position == 0) {
+            continue;
+        }
+        let scalar_index =
+            super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?;
+        let rtree_index = scalar_index
+            .as_any()
+            .downcast_ref::<RTreeIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "merge_existing_index_segments: expected RTree segment {}, got {:?}",
+                    segment.uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push(Arc::new(rtree_index.clone()));
+        source_filters.push(filter.clone());
+    }
+
+    let new_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
+    let created_index = lance_index::scalar::rtree::merge_rtree_indices(
+        &source_indices,
+        &new_store,
+        &source_filters,
+    )
+    .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar/zonemap.rs b/rust/lance/src/index/scalar/zonemap.rs
index 0cbd98f2c40..a3524b4955e 100644
--- a/rust/lance/src/index/scalar/zonemap.rs
+++ b/rust/lance/src/index/scalar/zonemap.rs
@@ -4,6 +4,7 @@
 use std::sync::Arc;
 
 use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::index_files_to_table;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::zonemap::ZoneMapIndex;
 use lance_table::format::IndexMetadata;
@@ -80,7 +81,7 @@ pub(in crate::index) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(created_index.files),
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
diff --git a/rust/lance/src/index/scalar_logical.rs b/rust/lance/src/index/scalar_logical.rs
index 4ff0da7f091..d6b30188573 100644
--- a/rust/lance/src/index/scalar_logical.rs
+++ b/rust/lance/src/index/scalar_logical.rs
@@ -23,6 +23,21 @@ use crate::dataset::Dataset;
 use crate::index::scalar::fetch_index_details;
 use crate::index::{DatasetIndexExt, DatasetIndexInternalExt};
 
+/// Query-time view that exposes several physical scalar index segments as a single [`ScalarIndex`].
+///
+/// A named scalar index can be built incrementally, producing multiple physical segments that
+/// each cover a disjoint set of fragments. When such an index is opened, the loader bundles
+/// the segments into a `LogicalScalarIndex` so the scanner can treat them as one index: queries
+/// are fanned out to every segment in parallel and the row-address results are unioned together.
+///
+/// All segments must share the same [`IndexType`]; mixing types is rejected at construction.
+/// Per-segment [`SearchResult`] precision is preserved when combining: a union of `Exact`
+/// results stays `Exact`, a union containing `AtMost` results yields `AtMost`, and a union
+/// containing `AtLeast` results yields `AtLeast`. Combining `AtMost` and `AtLeast` segments in
+/// the same query is not supported.
+///
+/// This is a read-only wrapper. [`ScalarIndex::remap`] and [`ScalarIndex::update`] both return
+/// an error — callers must rebuild the index to consolidate segments before mutating it.
 #[derive(Debug)]
 pub struct LogicalScalarIndex {
     name: String,
@@ -136,6 +151,12 @@ impl ScalarIndex for LogicalScalarIndex {
         combine_search_results(results)
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        // All segments of a logical index share the same underlying index type,
+        // so they agree on the result domain.
+        self.segments[0].results_are_row_addresses()
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
@@ -274,6 +295,11 @@ fn union_fragment_bitmaps(indices: &[IndexMetadata], index_name: &str) -> Result
     Ok(combined)
 }
 
+/// Return the union of fragment bitmaps across every usable segment of a named scalar index.
+///
+/// Only segments whose fragment bitmap intersects the dataset's current fragment set are
+/// considered. Returns `Ok(None)` when no such segment exists, `Ok(Some(bitmap))` otherwise.
+/// Errors if the segments disagree on their underlying index type.
 pub async fn scalar_index_fragment_bitmap(
     dataset: &Dataset,
     column: &str,
@@ -290,6 +316,13 @@ pub async fn scalar_index_fragment_bitmap(
     }
 }
 
+/// Open a named scalar index, transparently bundling multiple segments when present.
+///
+/// Loads every segment registered under `index_name` whose fragment bitmap intersects the
+/// dataset. If exactly one usable segment exists it is returned directly; if multiple exist
+/// they are wrapped in a [`LogicalScalarIndex`] so the caller sees a single [`ScalarIndex`].
+/// Errors if no usable segment exists (the scanner planned a query against an index that is
+/// not present) or if the segments mix incompatible types.
 pub async fn open_named_scalar_index(
     dataset: &Dataset,
     column: &str,
diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs
index c9920fb8adf..bfc4a4f3474 100644
--- a/rust/lance/src/index/vector.rs
+++ b/rust/lance/src/index/vector.rs
@@ -19,6 +19,7 @@ pub mod utils;
 mod fixture_test;
 
 use self::{ivf::*, pq::PQIndex};
+use arrow_array::Array;
 use arrow_schema::{DataType, Schema};
 use builder::{IvfIndexBuilder, VectorIndexBuildSummary};
 use datafusion::physical_plan::SendableRecordBatchStream;
@@ -516,6 +517,7 @@ async fn prepare_vector_segment_build(
     progress: Arc<dyn IndexBuildProgress>,
     mode: &str,
     require_precomputed_ivf: bool,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<(DataType, IndexType, IvfBuildParams, Box<dyn Shuffler>)> {
     let stages = &params.stages;
 
@@ -557,15 +559,29 @@ async fn prepare_vector_segment_build(
         validate_supported_rq_num_bits(rq_params.num_bits)?;
     }
 
-    let num_rows = dataset.count_rows(None).await?;
-    let num_partitions = ivf_params0.num_partitions.unwrap_or_else(|| {
-        recommended_num_partitions(
-            num_rows,
-            ivf_params0
-                .target_partition_size
-                .unwrap_or(index_type.target_partition_size()),
-        )
-    });
+    let num_partitions = match (ivf_params0.num_partitions, ivf_params0.centroids.as_ref()) {
+        (Some(num_partitions), Some(centroids)) if num_partitions != centroids.len() => {
+            return Err(Error::index(format!(
+                "{mode}: num_partitions {} does not match precomputed IVF centroids length {}",
+                num_partitions,
+                centroids.len()
+            )));
+        }
+        (Some(num_partitions), _) => num_partitions,
+        (None, Some(centroids)) => centroids.len(),
+        (None, None) => {
+            let num_rows = match fragment_ids {
+                Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await?,
+                None => dataset.count_rows(None).await?,
+            };
+            recommended_num_partitions(
+                num_rows,
+                ivf_params0
+                    .target_partition_size
+                    .unwrap_or(index_type.target_partition_size()),
+            )
+        }
+    };
     let mut ivf_params = ivf_params0.clone();
     ivf_params.num_partitions = Some(num_partitions);
 
@@ -602,6 +618,7 @@ pub(crate) async fn build_distributed_vector_index(
         progress.clone(),
         "Build Distributed Vector Index",
         true,
+        Some(fragment_ids),
     )
     .await?;
     let stages = &params.stages;
@@ -945,6 +962,55 @@ pub(crate) async fn build_vector_index(
     params: &VectorIndexParams,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
     progress: Arc<dyn IndexBuildProgress>,
+) -> Result<Vec<IndexFile>> {
+    build_vector_index_impl(
+        dataset,
+        column,
+        name,
+        uuid,
+        params,
+        frag_reuse_index,
+        progress,
+        None,
+    )
+    .await
+}
+
+/// Build a standalone vector index segment over a subset of fragments.
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn build_filtered_vector_index(
+    dataset: &Dataset,
+    column: &str,
+    name: &str,
+    uuid: Uuid,
+    params: &VectorIndexParams,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    fragment_ids: &[u32],
+    progress: Arc<dyn IndexBuildProgress>,
+) -> Result<Vec<IndexFile>> {
+    build_vector_index_impl(
+        dataset,
+        column,
+        name,
+        uuid,
+        params,
+        frag_reuse_index,
+        progress,
+        Some(fragment_ids),
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn build_vector_index_impl(
+    dataset: &Dataset,
+    column: &str,
+    name: &str,
+    uuid: Uuid,
+    params: &VectorIndexParams,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    progress: Arc<dyn IndexBuildProgress>,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<Vec<IndexFile>> {
     let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build(
         dataset,
@@ -953,6 +1019,7 @@ pub(crate) async fn build_vector_index(
         progress.clone(),
         "Build Vector Index",
         false,
+        fragment_ids,
     )
     .await?;
     let stages = &params.stages;
@@ -971,10 +1038,11 @@ pub(crate) async fn build_vector_index(
                     (),
                     frag_reuse_index,
                 )?
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
-                return Ok(summary.files);
+                Ok(summary.files)
             }
             DataType::UInt8 => {
                 let summary = IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
@@ -988,17 +1056,16 @@ pub(crate) async fn build_vector_index(
                     (),
                     frag_reuse_index,
                 )?
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
-                return Ok(summary.files);
-            }
-            _ => {
-                return Err(Error::index(format!(
-                    "Build Vector Index: invalid data type: {:?}",
-                    element_type
-                )));
+                Ok(summary.files)
             }
+            _ => Err(Error::index(format!(
+                "Build Vector Index: invalid data type: {:?}",
+                element_type
+            ))),
         },
         IndexType::IvfPq => {
             let len = stages.len();
@@ -1011,6 +1078,12 @@ pub(crate) async fn build_vector_index(
 
             match params.version {
                 IndexFileVersion::Legacy => {
+                    if fragment_ids.is_some() {
+                        return Err(Error::index(
+                            "Build Vector Index: filtered IVF_PQ builds do not support legacy format"
+                                .to_string(),
+                        ));
+                    }
                     let files = build_ivf_pq_index(
                         dataset,
                         column,
@@ -1022,7 +1095,7 @@ pub(crate) async fn build_vector_index(
                         progress.clone(),
                     )
                     .await?;
-                    return Ok(files);
+                    Ok(files)
                 }
                 IndexFileVersion::V3 => {
                     let mut builder = IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new(
@@ -1039,10 +1112,11 @@ pub(crate) async fn build_vector_index(
 
                     let summary = builder
                         .with_transpose(!params.skip_transpose)
+                        .with_optional_fragment_filter(fragment_ids)
                         .with_progress(progress.clone())
                         .build()
                         .await?;
-                    return Ok(summary.files);
+                    Ok(summary.files)
                 }
             }
         }
@@ -1065,10 +1139,11 @@ pub(crate) async fn build_vector_index(
                 (),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
-            return Ok(summary.files);
+            Ok(summary.files)
         }
         IndexType::IvfRq => {
             let StageParams::RQ(rq_params) = &stages[1] else {
@@ -1092,10 +1167,11 @@ pub(crate) async fn build_vector_index(
 
             let summary = builder
                 .with_transpose(!params.skip_transpose)
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
-            return Ok(summary.files);
+            Ok(summary.files)
         }
         IndexType::IvfHnswFlat => {
             let StageParams::Hnsw(hnsw_params) = &stages[1] else {
@@ -1117,10 +1193,11 @@ pub(crate) async fn build_vector_index(
                         hnsw_params.clone(),
                         frag_reuse_index,
                     )?
+                    .with_optional_fragment_filter(fragment_ids)
                     .with_progress(progress.clone())
                     .build()
                     .await?;
-                    return Ok(summary.files);
+                    Ok(summary.files)
                 }
                 _ => {
                     let summary = IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
@@ -1134,10 +1211,11 @@ pub(crate) async fn build_vector_index(
                         hnsw_params.clone(),
                         frag_reuse_index,
                     )?
+                    .with_optional_fragment_filter(fragment_ids)
                     .with_progress(progress.clone())
                     .build()
                     .await?;
-                    return Ok(summary.files);
+                    Ok(summary.files)
                 }
             }
         }
@@ -1165,10 +1243,11 @@ pub(crate) async fn build_vector_index(
                 hnsw_params.clone(),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
-            return Ok(summary.files);
+            Ok(summary.files)
         }
         IndexType::IvfHnswSq => {
             let StageParams::Hnsw(hnsw_params) = &stages[1] else {
@@ -1194,17 +1273,16 @@ pub(crate) async fn build_vector_index(
                 hnsw_params.clone(),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
-            return Ok(summary.files);
-        }
-        _ => {
-            return Err(Error::index(format!(
-                "Build Vector Index: invalid index type: {:?}",
-                index_type
-            )));
+            Ok(summary.files)
         }
+        _ => Err(Error::index(format!(
+            "Build Vector Index: invalid index type: {:?}",
+            index_type
+        ))),
     }
 }
 
diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs
index bf968d9744f..f881d28e745 100644
--- a/rust/lance/src/index/vector/builder.rs
+++ b/rust/lance/src/index/vector/builder.rs
@@ -395,7 +395,14 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
 
     /// Set fragment filter for distributed indexing
     pub fn with_fragment_filter(&mut self, fragment_ids: Vec<u32>) -> &mut Self {
-        self.fragment_filter = Some(fragment_ids);
+        self.fragment_filter = Some(Dataset::normalize_fragment_ids(&fragment_ids));
+        self
+    }
+
+    pub fn with_optional_fragment_filter(&mut self, fragment_ids: Option<&[u32]>) -> &mut Self {
+        if let Some(fragment_ids) = fragment_ids {
+            self.fragment_filter = Some(Dataset::normalize_fragment_ids(fragment_ids));
+        }
         self
     }
 
@@ -553,19 +560,11 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             return Ok(None);
         };
         match &self.fragment_filter {
-            Some(fragment_ids) => {
-                let fragments: Vec<_> = dataset
-                    .get_fragments()
-                    .into_iter()
-                    .filter(|f| fragment_ids.contains(&(f.id() as u32)))
-                    .collect();
-                let counts = futures::stream::iter(fragments)
-                    .map(|f| async move { f.count_rows(None).await })
-                    .buffer_unordered(16) // ref: Dataset::count_all_rows()
-                    .try_collect::<Vec<_>>()
-                    .await?;
-                Ok(Some(counts.iter().sum::<usize>() as u64))
-            }
+            Some(fragment_ids) => Ok(Some(
+                dataset
+                    .count_rows_in_existing_fragments(fragment_ids)
+                    .await? as u64,
+            )),
             None => Ok(Some(dataset.count_rows(None).await? as u64)),
         }
     }
@@ -603,14 +602,9 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                         "applying fragment filter for distributed indexing: {:?}",
                         fragment_ids
                     );
-                    // Filter fragments by converting fragment_ids to Fragment objects
-                    let all_fragments = dataset.fragments();
-                    let filtered_fragments: Vec<_> = all_fragments
-                        .iter()
-                        .filter(|fragment| fragment_ids.contains(&(fragment.id as u32)))
-                        .cloned()
-                        .collect();
-                    builder.with_fragments(filtered_fragments);
+                    builder.with_fragments(
+                        dataset.get_existing_fragment_metadata_from_ids(fragment_ids),
+                    );
                 }
 
                 let (vector_type, _) = get_vector_type(dataset.schema(), &self.column)?;
diff --git a/rust/lance/src/index/vector/fixture_test.rs b/rust/lance/src/index/vector/fixture_test.rs
index a03cc313466..2af020e1df2 100644
--- a/rust/lance/src/index/vector/fixture_test.rs
+++ b/rust/lance/src/index/vector/fixture_test.rs
@@ -278,6 +278,7 @@ mod test {
                     deleted_ids: None,
                     filtered_ids: None,
                     deleted_fragments: None,
+                    overlay_block: None,
                     final_mask: Mutex::new(OnceCell::new()),
                 }),
                 &NoOpMetricsCollector,
diff --git a/rust/lance/src/index/vector/hamming.rs b/rust/lance/src/index/vector/hamming.rs
index ba6ea98c42d..3240ab3021b 100644
--- a/rust/lance/src/index/vector/hamming.rs
+++ b/rust/lance/src/index/vector/hamming.rs
@@ -5,123 +5,229 @@
 //!
 //! This module provides functionality to perform pairwise hamming distance
 //! computation and clustering on specific partitions of IVF_FLAT indices.
-
+//!
+//! A logical IVF_FLAT index may consist of multiple physical segments (e.g.
+//! delta segments created by `optimize_indices` in append mode, or segments
+//! committed by distributed index builds). All segments of one logical index
+//! are assumed to share the same global IVF centroids, so one partition id
+//! refers to the same centroid region in every segment; this is validated and
+//! an error is returned if the centroids differ.
+
+use std::sync::Arc;
 use std::time::Instant;
 
-use arrow_array::RecordBatchReader;
 use arrow_array::cast::AsArray;
 use arrow_array::types::UInt64Type;
+use arrow_array::{Array, FixedSizeListArray, RecordBatchReader};
 use arrow_schema::DataType;
 use lance_core::{Error, Result};
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::vector::VectorIndex;
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex};
 use lance_index::vector::flat::storage::FLAT_COLUMN;
+use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::storage::VectorStore;
 use lance_linalg::distance::{
-    ClusteringResult, cluster_pairwise_result, extract_hashes_from_fixed_list,
+    BinaryHashValues, ClusteringResult, cluster_pairwise_result,
+    extract_binary_hashes_from_fixed_list, pairwise_hamming_distance_binary_parallel,
     pairwise_hamming_distance_parallel,
 };
+use lance_table::format::IndexMetadata;
 use rand::rng;
 use rand::seq::index::sample;
+use uuid::Uuid;
 
 use crate::dataset::Dataset;
-use crate::index::{DatasetIndexExt, DatasetIndexInternalExt};
+use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, filter_index_segments_by_ids};
 
 use super::ivf::v2::IVFIndex;
 
-/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index.
-///
-/// This function loads a specific partition from an IVF_FLAT index on a hash column,
-/// computes pairwise hamming distances between all hashes in the partition,
-/// filters by threshold, and clusters the results using union-find.
-///
-/// # Arguments
-///
-/// * `dataset` - The Lance dataset
-/// * `index_name` - Name of the IVF_FLAT index on the hash column
-/// * `partition_id` - The partition ID within the IVF_FLAT index
-/// * `hamming_threshold` - Maximum hamming distance to consider as similar
-///
-/// # Returns
-///
-/// A `RecordBatchReader` yielding batches with columns:
-/// - `representative`: UInt64 - The representative row ID for each cluster
-/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
-///
-/// # Errors
+/// One opened physical segment of a logical IVF_FLAT binary index.
+struct HashIndexSegment {
+    metadata: IndexMetadata,
+    index: Arc<dyn VectorIndex>,
+}
+
+impl HashIndexSegment {
+    fn ivf_flat_bin(&self) -> &IVFIndex<FlatIndex, FlatBinQuantizer> {
+        self.index
+            .as_any()
+            .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
+            .expect("segment type validated in open_hash_index_segments")
+    }
+}
+
+/// Validate that a column stores fixed-width binary hashes as
+/// `FixedSizeList<UInt8, N>`, where `N` is a positive multiple of 8 bytes.
+fn validate_hash_column(column: &str, data_type: &DataType) -> Result<usize> {
+    match data_type {
+        DataType::FixedSizeList(inner, size) if *inner.data_type() == DataType::UInt8 => {
+            if *size <= 0 || !(*size as usize).is_multiple_of(8) {
+                return Err(Error::invalid_input(format!(
+                    "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+                     multiple of 8 bytes, got FixedSizeList<UInt8, {}>",
+                    column, size
+                )));
+            }
+            Ok(*size as usize)
+        }
+        DataType::FixedSizeList(inner, size) => Err(Error::invalid_input(format!(
+            "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+             multiple of 8 bytes, got FixedSizeList<{:?}, {}>",
+            column,
+            inner.data_type(),
+            size
+        ))),
+        _ => Err(Error::invalid_input(format!(
+            "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+             multiple of 8 bytes, got {:?}",
+            column, data_type
+        ))),
+    }
+}
+
+/// Validate that every segment of a logical IVF index shares the same global
+/// centroids, so one partition id refers to the same centroid region in each
+/// segment. Fails if any segment has no centroids or diverging centroids.
+fn validate_shared_centroids<'a>(
+    index_name: &str,
+    models: impl IntoIterator<Item = (Uuid, &'a IvfModel)>,
+) -> Result<()> {
+    struct Reference<'a> {
+        uuid: Uuid,
+        num_partitions: usize,
+        centroids: &'a FixedSizeListArray,
+    }
+
+    let mut reference: Option<Reference<'a>> = None;
+    for (uuid, model) in models {
+        let centroids = model.centroids_array().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Index '{}' segment {} has no IVF centroids; hamming clustering requires \
+                 segments built from a shared global IVF model",
+                index_name, uuid
+            ))
+        })?;
+        match &reference {
+            None => {
+                reference = Some(Reference {
+                    uuid,
+                    num_partitions: model.num_partitions(),
+                    centroids,
+                });
+            }
+            Some(reference) => {
+                if centroids.to_data() != reference.centroids.to_data() {
+                    return Err(Error::invalid_input(format!(
+                        "Index '{}' segments do not share the same global IVF centroids: \
+                         segment {} ({} partitions) differs from segment {} ({} partitions); \
+                         retrain the index to merge segments before hamming clustering",
+                        index_name,
+                        uuid,
+                        model.num_partitions(),
+                        reference.uuid,
+                        reference.num_partitions
+                    )));
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Open the physical segments of a logical IVF_FLAT binary index.
 ///
-/// Returns an error if:
-/// - The index doesn't exist or is not an IVF_FLAT index
-/// - The indexed column has wrong type (must be `FixedSizeList<UInt8, 8>`)
-/// - The partition ID is out of range
-pub async fn hamming_clustering_for_ivf_partition(
+/// When `segment_ids` is `None` all segments are opened; otherwise only the
+/// requested segments are opened and every requested id must exist. Validates
+/// that all selected segments index the same fixed-width binary hash column,
+/// are IVF_FLAT indices for binary data, and share the same global centroids.
+async fn open_hash_index_segments(
     dataset: &Dataset,
     index_name: &str,
-    partition_id: usize,
-    hamming_threshold: u32,
-) -> Result<Box<dyn RecordBatchReader + Send>> {
-    // Load indices and find the IVF_FLAT index
-    let indices = dataset.load_indices().await?;
-    let index_meta = indices
-        .iter()
-        .find(|idx| idx.name == index_name)
-        .ok_or_else(|| {
-            Error::invalid_input(format!("Index '{}' not found on dataset", index_name))
-        })?;
+    segment_ids: Option<&[Uuid]>,
+) -> Result<Vec<HashIndexSegment>> {
+    let metadatas = dataset.load_indices_by_name(index_name).await?;
+    if metadatas.is_empty() {
+        return Err(Error::invalid_input(format!(
+            "Index '{}' not found on dataset",
+            index_name
+        )));
+    }
 
-    // Get the column name from the index metadata
-    let schema = dataset.schema();
-    let field_id = index_meta
-        .fields
+    let metadatas = match segment_ids {
+        None => metadatas,
+        Some(segment_ids) => {
+            if segment_ids.is_empty() {
+                return Err(Error::invalid_input(format!(
+                    "Segment selection for index '{}' must not be empty; \
+                     omit index_segments to use all segments",
+                    index_name
+                )));
+            }
+            filter_index_segments_by_ids(index_name, metadatas, segment_ids)?
+        }
+    };
+
+    let fields = metadatas[0].fields.clone();
+    if let Some(mismatched) = metadatas.iter().find(|meta| meta.fields != fields) {
+        return Err(Error::invalid_input(format!(
+            "Index '{}' segments cover different fields: segment {} covers {:?} \
+             while segment {} covers {:?}",
+            index_name, metadatas[0].uuid, fields, mismatched.uuid, mismatched.fields
+        )));
+    }
+
+    let field_id = fields
         .first()
         .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?;
+    let schema = dataset.schema();
     let field = schema.field_by_id(*field_id).ok_or_else(|| {
         Error::invalid_input(format!(
             "Field with id {} not found in schema for index '{}'",
             field_id, index_name
         ))
     })?;
-    let column = &field.name;
+    validate_hash_column(&field.name, &field.data_type())?;
 
-    // Check column is FixedSizeList<UInt8, 8>
-    let data_type = field.data_type();
-    match data_type {
-        DataType::FixedSizeList(inner, 8) => {
-            if *inner.data_type() != DataType::UInt8 {
-                return Err(Error::invalid_input(format!(
-                    "Column '{}' must be FixedSizeList<UInt8, 8>, got FixedSizeList<{:?}, 8>",
-                    column,
-                    inner.data_type()
-                )));
-            }
-        }
-        _ => {
+    let mut segments = Vec::with_capacity(metadatas.len());
+    for metadata in metadatas {
+        let index = dataset
+            .open_vector_index(&field.name, &metadata.uuid, &NoOpMetricsCollector)
+            .await?;
+        if index
+            .as_any()
+            .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
+            .is_none()
+        {
             return Err(Error::invalid_input(format!(
-                "Column '{}' must be FixedSizeList<UInt8, 8>, got {:?}",
-                column, data_type
+                "Index '{}' segment {} is not an IVF_FLAT index for binary data",
+                index_name, metadata.uuid
             )));
         }
+        segments.push(HashIndexSegment { metadata, index });
     }
 
-    // Open the vector index
-    let index = dataset
-        .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector)
-        .await?;
+    validate_shared_centroids(
+        index_name,
+        segments
+            .iter()
+            .map(|segment| (segment.metadata.uuid, segment.ivf_flat_bin().ivf_model())),
+    )?;
 
-    // Try to downcast to IVFIndex<FlatIndex, FlatBinQuantizer> (IVF_FLAT for binary data)
-    let ivf_index = index
-        .as_any()
-        .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
-        .ok_or_else(|| {
-            Error::invalid_input(format!(
-                "Index '{}' is not an IVF_FLAT index for binary data",
-                index_name
-            ))
-        })?;
+    Ok(segments)
+}
 
-    // Check partition ID is valid
-    let num_partitions = ivf_index.ivf_model().num_partitions();
+async fn hamming_clustering_for_ivf_partition_impl(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: Option<&[Uuid]>,
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?;
+
+    // All segments share centroids, so the partition count is uniform.
+    let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions();
     if partition_id >= num_partitions {
         return Err(Error::invalid_input(format!(
             "Partition ID {} is out of range (0..{})",
@@ -129,45 +235,53 @@ pub async fn hamming_clustering_for_ivf_partition(
         )));
     }
 
-    // Load the partition storage
-    let storage = ivf_index.load_partition_storage(partition_id, None).await?;
-
-    // Get row IDs
-    let row_id_slice: Vec<u64> = storage.row_ids().copied().collect();
-
-    if row_id_slice.is_empty() {
-        let empty = ClusteringResult {
-            clusters: Vec::new(),
-        };
-        return Ok(empty.into_reader(None));
+    // Concatenate the partition's row ids and hashes across segments; identical
+    // hashes land in the same partition of every segment, so one pairwise pass
+    // over the union finds cross-segment duplicates.
+    let mut all_row_ids: Vec<u64> = Vec::new();
+    let mut hash_chunks = Vec::new();
+    let mut num_hashes = 0;
+    for segment in &segments {
+        let storage = segment
+            .ivf_flat_bin()
+            .load_partition_storage(partition_id, None)
+            .await?;
+        all_row_ids.extend(storage.row_ids().copied());
+        for batch in storage.to_batches()? {
+            let vectors = batch
+                .column_by_name(FLAT_COLUMN)
+                .ok_or_else(|| {
+                    Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN))
+                })?
+                .as_fixed_size_list();
+            let hashes = extract_binary_hashes_from_fixed_list(vectors)?;
+            num_hashes += hashes.len();
+            hash_chunks.push(hashes);
+        }
+        if all_row_ids.len() != num_hashes {
+            return Err(Error::internal(format!(
+                "Index '{}' segment {} partition {}: row id count {} does not match hash count {}",
+                index_name,
+                segment.metadata.uuid,
+                partition_id,
+                all_row_ids.len(),
+                num_hashes
+            )));
+        }
     }
 
-    // Get vectors from the storage batches
-    let batches: Vec<_> = storage.to_batches()?.collect();
-    if batches.is_empty() {
+    if all_row_ids.is_empty() {
         let empty = ClusteringResult {
             clusters: Vec::new(),
         };
         return Ok(empty.into_reader(None));
     }
-
-    // Extract the hash vectors from the FLAT_COLUMN
-    let mut all_hashes = Vec::new();
-    for batch in &batches {
-        let vectors = batch
-            .column_by_name(FLAT_COLUMN)
-            .ok_or_else(|| {
-                Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN))
-            })?
-            .as_fixed_size_list();
-        let hashes = extract_hashes_from_fixed_list(vectors)?;
-        all_hashes.extend(hashes);
-    }
+    let all_hashes = BinaryHashValues::concat(&hash_chunks)?;
 
     // Compute pairwise hamming distances with threshold filtering
-    let pairwise_result = pairwise_hamming_distance_parallel(
+    let pairwise_result = pairwise_hamming_distance_binary_parallel(
         &all_hashes,
-        Some(&row_id_slice),
+        Some(&all_row_ids),
         Some(hamming_threshold),
     );
 
@@ -177,60 +291,125 @@ pub async fn hamming_clustering_for_ivf_partition(
     Ok(clustering.into_reader(None))
 }
 
-/// Get partition statistics for an IVF_FLAT index.
-pub async fn get_ivf_partition_info(
+/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index.
+///
+/// This function loads a specific partition from every segment of an IVF_FLAT
+/// index on a hash column, computes pairwise hamming distances between all
+/// hashes in the combined partition, filters by threshold, and clusters the
+/// results using union-find. See [`hamming_clustering_for_ivf_partition_segments`]
+/// to restrict the computation to selected segments.
+///
+/// # Arguments
+///
+/// * `dataset` - The Lance dataset
+/// * `index_name` - Name of the IVF_FLAT index on the hash column
+/// * `partition_id` - The partition ID within the IVF_FLAT index
+/// * `hamming_threshold` - Maximum hamming distance to consider as similar
+///
+/// # Returns
+///
+/// A `RecordBatchReader` yielding batches with columns:
+/// - `representative`: UInt64 - The representative row ID for each cluster
+/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The index doesn't exist or is not an IVF_FLAT index
+/// - The indexed column has wrong type (must be `FixedSizeList<UInt8, N>` where
+///   `N` is a positive multiple of 8 bytes)
+/// - The index segments do not share the same global IVF centroids
+/// - The partition ID is out of range
+pub async fn hamming_clustering_for_ivf_partition(
     dataset: &Dataset,
     index_name: &str,
-) -> Result<Vec<PartitionInfo>> {
-    let indices = dataset.load_indices().await?;
-    let index_meta = indices
-        .iter()
-        .find(|idx| idx.name == index_name)
-        .ok_or_else(|| {
-            Error::invalid_input(format!("Index '{}' not found on dataset", index_name))
-        })?;
-
-    // Get the column name from the index metadata
-    let schema = dataset.schema();
-    let field_id = index_meta
-        .fields
-        .first()
-        .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?;
-    let field = schema.field_by_id(*field_id).ok_or_else(|| {
-        Error::invalid_input(format!(
-            "Field with id {} not found in schema for index '{}'",
-            field_id, index_name
-        ))
-    })?;
-    let column = &field.name;
-
-    let index = dataset
-        .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector)
-        .await?;
-
-    let ivf_index = index
-        .as_any()
-        .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
-        .ok_or_else(|| {
-            Error::invalid_input(format!(
-                "Index '{}' is not an IVF_FLAT index for binary data",
-                index_name
-            ))
-        })?;
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    hamming_clustering_for_ivf_partition_impl(
+        dataset,
+        index_name,
+        None,
+        partition_id,
+        hamming_threshold,
+    )
+    .await
+}
 
-    let num_partitions = ivf_index.ivf_model().num_partitions();
-    let mut partition_infos = Vec::with_capacity(num_partitions);
+/// Perform pairwise hamming distance clustering on a partition of selected
+/// segments of an IVF_FLAT index.
+///
+/// Same as [`hamming_clustering_for_ivf_partition`] but only the requested
+/// physical segments contribute rows. Segment ids are the index UUIDs reported
+/// by index descriptions; every requested id must belong to the named index and
+/// the selection must not be empty.
+pub async fn hamming_clustering_for_ivf_partition_segments(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: &[Uuid],
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    hamming_clustering_for_ivf_partition_impl(
+        dataset,
+        index_name,
+        Some(segment_ids),
+        partition_id,
+        hamming_threshold,
+    )
+    .await
+}
 
-    for i in 0..num_partitions {
-        partition_infos.push(PartitionInfo {
-            partition_id: i,
-            size: ivf_index.ivf_model().partition_size(i),
-        });
+async fn get_ivf_partition_info_impl(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: Option<&[Uuid]>,
+) -> Result<Vec<PartitionInfo>> {
+    let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?;
+
+    let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions();
+    let mut partition_infos: Vec<PartitionInfo> = (0..num_partitions)
+        .map(|partition_id| PartitionInfo {
+            partition_id,
+            size: 0,
+        })
+        .collect();
+    for segment in &segments {
+        // Sizes come from the partition storage; the IVF model of a v3 index
+        // file does not carry partition lengths.
+        let index = segment.ivf_flat_bin();
+        for info in partition_infos.iter_mut() {
+            info.size += index.partition_size(info.partition_id);
+        }
     }
 
     Ok(partition_infos)
 }
 
+/// Get partition statistics for an IVF_FLAT index.
+///
+/// Partition sizes are aggregated across all segments of the logical index.
+/// See [`get_ivf_partition_info_segments`] to restrict the statistics to
+/// selected segments.
+pub async fn get_ivf_partition_info(
+    dataset: &Dataset,
+    index_name: &str,
+) -> Result<Vec<PartitionInfo>> {
+    get_ivf_partition_info_impl(dataset, index_name, None).await
+}
+
+/// Get partition statistics for selected segments of an IVF_FLAT index.
+///
+/// Same as [`get_ivf_partition_info`] but only the requested physical segments
+/// contribute to the partition sizes.
+pub async fn get_ivf_partition_info_segments(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: &[Uuid],
+) -> Result<Vec<PartitionInfo>> {
+    get_ivf_partition_info_impl(dataset, index_name, Some(segment_ids)).await
+}
+
 /// Information about an IVF partition.
 #[derive(Debug, Clone)]
 pub struct PartitionInfo {
@@ -247,7 +426,8 @@ pub struct PartitionInfo {
 /// # Arguments
 ///
 /// * `dataset` - The Lance dataset
-/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, 8>`)
+/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, N>`
+///   where `N` is a positive multiple of 8 bytes)
 /// * `sample_size` - Number of rows to sample (if None or >= total rows, uses all rows)
 /// * `hamming_threshold` - Maximum hamming distance to consider as similar
 ///
@@ -267,26 +447,7 @@ pub async fn hamming_clustering_for_sample(
     let field = schema.field(column).ok_or_else(|| {
         Error::invalid_input(format!("Column '{}' not found in dataset schema", column))
     })?;
-
-    // Check column is FixedSizeList<UInt8, 8>
-    let data_type = field.data_type();
-    match data_type {
-        DataType::FixedSizeList(inner, 8) => {
-            if *inner.data_type() != DataType::UInt8 {
-                return Err(Error::invalid_input(format!(
-                    "Column '{}' must be FixedSizeList<UInt8, 8>, got FixedSizeList<{:?}, 8>",
-                    column,
-                    inner.data_type()
-                )));
-            }
-        }
-        _ => {
-            return Err(Error::invalid_input(format!(
-                "Column '{}' must be FixedSizeList<UInt8, 8>, got {:?}",
-                column, data_type
-            )));
-        }
-    }
+    validate_hash_column(column, &field.data_type())?;
 
     // Get total row count
     let total_rows: usize = dataset
@@ -326,7 +487,7 @@ pub async fn hamming_clustering_for_sample(
             Error::invalid_input(format!("Column '{}' not found in result", column))
         })?;
         let hashes_arr = hash_col.as_fixed_size_list();
-        let hashes = extract_hashes_from_fixed_list(hashes_arr)?;
+        let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
 
         (hashes, row_id_vec)
     } else {
@@ -348,7 +509,7 @@ pub async fn hamming_clustering_for_sample(
             Error::invalid_input(format!("Column '{}' not found in result", column))
         })?;
         let hashes_arr = hash_col.as_fixed_size_list();
-        let hashes = extract_hashes_from_fixed_list(hashes_arr)?;
+        let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
 
         (hashes, row_id_vec)
     };
@@ -362,7 +523,7 @@ pub async fn hamming_clustering_for_sample(
 
     // Compute pairwise hamming distances
     let pairwise =
-        pairwise_hamming_distance_parallel(&hashes, Some(&row_ids), Some(hamming_threshold));
+        pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(hamming_threshold));
 
     // Cluster edges
     let clustering = cluster_pairwise_result(&pairwise);
@@ -380,7 +541,8 @@ pub async fn hamming_clustering_for_sample(
 /// # Arguments
 ///
 /// * `dataset` - The Lance dataset
-/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, 8>`)
+/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, N>`
+///   where `N` is a positive multiple of 8 bytes)
 /// * `fragment_id` - The fragment ID to read from
 /// * `start_row` - The starting row offset within the fragment
 /// * `num_rows` - Number of rows to read from the start position
@@ -396,7 +558,8 @@ pub async fn hamming_clustering_for_sample(
 ///
 /// Returns an error if:
 /// - The fragment doesn't exist
-/// - The column has wrong type (must be `FixedSizeList<UInt8, 8>`)
+/// - The column has wrong type (must be `FixedSizeList<UInt8, N>` where `N`
+///   is a positive multiple of 8 bytes)
 /// - The row range is out of bounds
 pub async fn hamming_clustering_for_range(
     dataset: &Dataset,
@@ -411,26 +574,7 @@ pub async fn hamming_clustering_for_range(
     let field = schema.field(column).ok_or_else(|| {
         Error::invalid_input(format!("Column '{}' not found in dataset schema", column))
     })?;
-
-    // Check column is FixedSizeList<UInt8, 8>
-    let data_type = field.data_type();
-    match data_type {
-        DataType::FixedSizeList(inner, 8) => {
-            if *inner.data_type() != DataType::UInt8 {
-                return Err(Error::invalid_input(format!(
-                    "Column '{}' must be FixedSizeList<UInt8, 8>, got FixedSizeList<{:?}, 8>",
-                    column,
-                    inner.data_type()
-                )));
-            }
-        }
-        _ => {
-            return Err(Error::invalid_input(format!(
-                "Column '{}' must be FixedSizeList<UInt8, 8>, got {:?}",
-                column, data_type
-            )));
-        }
-    }
+    validate_hash_column(column, &field.data_type())?;
 
     // Get the fragment
     let fragment = dataset.get_fragment(fragment_id).ok_or_else(|| {
@@ -483,7 +627,7 @@ pub async fn hamming_clustering_for_range(
         .column_by_name(column)
         .ok_or_else(|| Error::invalid_input(format!("Column '{}' not found in result", column)))?;
     let hashes_arr = hash_col.as_fixed_size_list();
-    let hashes = extract_hashes_from_fixed_list(hashes_arr)?;
+    let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
 
     if hashes.len() < 2 {
         let empty = ClusteringResult {
@@ -493,8 +637,11 @@ pub async fn hamming_clustering_for_range(
     }
 
     // Compute pairwise hamming distances
-    let pairwise =
-        pairwise_hamming_distance_parallel(&hashes, Some(&row_id_vec), Some(hamming_threshold));
+    let pairwise = pairwise_hamming_distance_binary_parallel(
+        &hashes,
+        Some(&row_id_vec),
+        Some(hamming_threshold),
+    );
 
     // Cluster edges
     let clustering = cluster_pairwise_result(&pairwise);
@@ -599,6 +746,31 @@ mod tests {
         clusters
     }
 
+    #[test]
+    fn test_validate_hash_column_generic_widths() {
+        use arrow_schema::{DataType, Field};
+        use std::sync::Arc;
+
+        fn hash_type(size: i32) -> DataType {
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::UInt8, true)), size)
+        }
+
+        assert_eq!(validate_hash_column("hash", &hash_type(8)).unwrap(), 8);
+        assert_eq!(validate_hash_column("hash", &hash_type(16)).unwrap(), 16);
+        assert_eq!(validate_hash_column("hash", &hash_type(32)).unwrap(), 32);
+
+        let err = validate_hash_column("hash", &hash_type(12)).unwrap_err();
+        assert!(err.to_string().contains("12"), "{}", err);
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+
+        let err = validate_hash_column("hash", &hash_type(4)).unwrap_err();
+        assert!(err.to_string().contains("4"), "{}", err);
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+
+        let err = validate_hash_column("hash", &hash_type(-8)).unwrap_err();
+        assert!(err.to_string().contains("-8"), "{}", err);
+    }
+
     #[test]
     fn test_hamming_clustering_from_hashes_basic() {
         // Create some test hashes with known distances
@@ -764,6 +936,200 @@ mod tests {
         assert!(err.to_string().contains("not found"), "Error: {}", err);
     }
 
+    #[test]
+    fn test_validate_shared_centroids() {
+        use arrow_array::UInt8Array;
+        use lance_arrow::FixedSizeListArrayExt;
+
+        fn model_from_bytes(bytes: Vec<u8>) -> IvfModel {
+            let centroids =
+                FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 8).unwrap();
+            IvfModel::new(centroids, None)
+        }
+
+        let uuid_a = Uuid::new_v4();
+        let uuid_b = Uuid::new_v4();
+
+        let model_a = model_from_bytes(vec![0u8; 16]);
+        let model_b = model_from_bytes(vec![0u8; 16]);
+        validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_b)]).unwrap();
+
+        let mut diverged = vec![0u8; 16];
+        diverged[0] = 1;
+        let model_c = model_from_bytes(diverged);
+        let err =
+            validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_c)]).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("do not share the same global IVF centroids"),
+            "{}",
+            err
+        );
+        assert!(err.to_string().contains(&uuid_b.to_string()), "{}", err);
+
+        let model_d = model_from_bytes(vec![0u8; 24]);
+        let err =
+            validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_d)]).unwrap_err();
+        assert!(err.to_string().contains("2 partitions"), "{}", err);
+        assert!(err.to_string().contains("3 partitions"), "{}", err);
+
+        let err = validate_shared_centroids("idx", [(uuid_a, &IvfModel::empty())]).unwrap_err();
+        assert!(err.to_string().contains("has no IVF centroids"), "{}", err);
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_ivf_partition_multi_segment_128_bit() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use lance_index::optimize::OptimizeOptions;
+        use lance_index::vector::ivf::IvfBuildParams;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        const HASH_BYTES: i32 = 16;
+
+        fn hash_batch(schema: Arc<Schema>, values: &[u64]) -> arrow_array::RecordBatch {
+            let mut bytes = Vec::with_capacity(values.len() * HASH_BYTES as usize);
+            for value in values {
+                bytes.extend_from_slice(&value.to_le_bytes());
+                let high = value.rotate_left(17) ^ 0xA5A5_A5A5_A5A5_A5A5;
+                bytes.extend_from_slice(&high.to_le_bytes());
+            }
+            let array =
+                FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), HASH_BYTES)
+                    .unwrap();
+            arrow_array::RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap()
+        }
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                HASH_BYTES,
+            ),
+            false,
+        )]));
+
+        // 25 distinct hash values, two copies each; the same batch is written to
+        // fragment 0 and appended as fragment 1, so every value has duplicates
+        // in both fragments.
+        let values: Vec<u64> = (0..50)
+            .map(|i| ((i / 2) as u64).wrapping_mul(0x9E3779B97F4A7C15))
+            .collect();
+        let num_values = 25;
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+        let reader = RecordBatchIterator::new(
+            vec![Ok(hash_batch(schema.clone(), &values))],
+            schema.clone(),
+        );
+        let mut dataset = crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        let params = crate::index::vector::VectorIndexParams::with_ivf_flat_params(
+            lance_linalg::distance::MetricType::Hamming,
+            IvfBuildParams::new(4),
+        );
+        dataset
+            .create_index(
+                &["hash"],
+                crate::index::IndexType::Vector,
+                Some("hash_idx".into()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        let reader = RecordBatchIterator::new(
+            vec![Ok(hash_batch(schema.clone(), &values))],
+            schema.clone(),
+        );
+        dataset.append(reader, None).await.unwrap();
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        let segments = dataset.load_indices_by_name("hash_idx").await.unwrap();
+        assert_eq!(
+            segments.len(),
+            2,
+            "expected a delta segment after optimize append"
+        );
+
+        // Partition sizes aggregate across both segments.
+        let infos = get_ivf_partition_info(&dataset, "hash_idx").await.unwrap();
+        assert_eq!(infos.len(), 4);
+        assert_eq!(infos.iter().map(|info| info.size).sum::<usize>(), 100);
+
+        // Clustering each partition with threshold 0 must group all four copies
+        // of every value, including the copies in the appended fragment.
+        const FRAG1_START: u64 = 1 << 32;
+        let mut clusters = Vec::new();
+        for partition_id in 0..4 {
+            let reader =
+                hamming_clustering_for_ivf_partition(&dataset, "hash_idx", partition_id, 0)
+                    .await
+                    .unwrap();
+            clusters.extend(collect_clusters(reader));
+        }
+        assert_eq!(clusters.len(), num_values);
+        for (representative, duplicates) in &clusters {
+            assert_eq!(duplicates.len(), 3);
+            assert!(*representative < FRAG1_START);
+            assert!(
+                duplicates.iter().any(|row_id| *row_id >= FRAG1_START),
+                "cluster {} should contain rows from the appended fragment",
+                representative
+            );
+        }
+
+        // Selecting only the original segment reproduces the single-segment scope.
+        let first_segment = segments
+            .iter()
+            .find(|meta| meta.fragment_bitmap.as_ref().unwrap().contains(0))
+            .unwrap();
+        let mut old_clusters = Vec::new();
+        for partition_id in 0..4 {
+            let reader = hamming_clustering_for_ivf_partition_segments(
+                &dataset,
+                "hash_idx",
+                &[first_segment.uuid],
+                partition_id,
+                0,
+            )
+            .await
+            .unwrap();
+            old_clusters.extend(collect_clusters(reader));
+        }
+        assert_eq!(old_clusters.len(), num_values);
+        for (representative, duplicates) in &old_clusters {
+            assert_eq!(duplicates.len(), 1);
+            assert!(*representative < FRAG1_START);
+            assert!(duplicates.iter().all(|row_id| *row_id < FRAG1_START));
+        }
+        let infos = get_ivf_partition_info_segments(&dataset, "hash_idx", &[first_segment.uuid])
+            .await
+            .unwrap();
+        assert_eq!(infos.iter().map(|info| info.size).sum::<usize>(), 50);
+
+        // Invalid segment selections are rejected.
+        let err = hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[], 0, 0)
+            .await
+            .err()
+            .unwrap();
+        assert!(err.to_string().contains("must not be empty"), "{}", err);
+        let missing = Uuid::new_v4();
+        let err =
+            hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[missing], 0, 0)
+                .await
+                .err()
+                .unwrap();
+        assert!(err.to_string().contains(&missing.to_string()), "{}", err);
+    }
+
     #[tokio::test]
     async fn test_hamming_clustering_for_sample_integration() {
         use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs
index 9b8db75fde3..1d56300fc4b 100644
--- a/rust/lance/src/index/vector/ivf.rs
+++ b/rust/lance/src/index/vector/ivf.rs
@@ -386,6 +386,106 @@ pub(crate) fn select_segment_for_single_rebalance(
     Ok(selected.map(|candidate| candidate.segment_id))
 }
 
+fn validate_shared_vector_model(indices: &[Arc<dyn VectorIndex>], operation: &str) -> Result<()> {
+    let Some(first) = indices.first() else {
+        return Ok(());
+    };
+    if indices.len() == 1 {
+        return Ok(());
+    }
+
+    let first_metric = first.metric_type();
+    let first_index_type = first.sub_index_type();
+    let first_centroids = first.ivf_model().centroids_array();
+    let first_quantizer = first.quantizer();
+    let first_quantizer_type = first_quantizer.quantization_type();
+
+    for (idx, index) in indices.iter().enumerate().skip(1) {
+        if index.metric_type() != first_metric {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has metric {:?}, expected {:?}",
+                index.metric_type(),
+                first_metric
+            )));
+        }
+        let index_type = index.sub_index_type();
+        if std::mem::discriminant(&index_type.0) != std::mem::discriminant(&first_index_type.0)
+            || index_type.1 != first_index_type.1
+        {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has type {:?}, expected {:?}",
+                index_type, first_index_type
+            )));
+        }
+        match (first_centroids, index.ivf_model().centroids_array()) {
+            (Some(expected), Some(actual)) if expected.to_data() != actual.to_data() => {
+                return Err(Error::index(format!(
+                    "{operation}: vector index segments do not share IVF centroids"
+                )));
+            }
+            (Some(_), None) | (None, Some(_)) => {
+                return Err(Error::index(format!(
+                    "{operation}: vector index segments do not share IVF centroids"
+                )));
+            }
+            _ => {}
+        }
+
+        let quantizer = index.quantizer();
+        if quantizer.quantization_type() != first_quantizer_type {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has quantizer {:?}, expected {:?}",
+                quantizer.quantization_type(),
+                first_quantizer_type
+            )));
+        }
+        if !shared_quantizer_model(&first_quantizer, &quantizer) {
+            return Err(Error::index(format!(
+                "{operation}: vector index segments do not share quantizer metadata"
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+fn shared_quantizer_model(left: &Quantizer, right: &Quantizer) -> bool {
+    match (left, right) {
+        (Quantizer::Flat(left), Quantizer::Flat(right)) => {
+            left.metadata(None).dim == right.metadata(None).dim
+        }
+        (Quantizer::FlatBin(left), Quantizer::FlatBin(right)) => {
+            left.metadata(None).dim == right.metadata(None).dim
+        }
+        (Quantizer::Product(left), Quantizer::Product(right)) => {
+            left.num_sub_vectors == right.num_sub_vectors
+                && left.num_bits == right.num_bits
+                && left.dimension == right.dimension
+                && left.distance_type == right.distance_type
+                && left.codebook.to_data() == right.codebook.to_data()
+        }
+        (Quantizer::Scalar(left), Quantizer::Scalar(right)) => {
+            left.metadata(None) == right.metadata(None)
+        }
+        (Quantizer::Rabit(left), Quantizer::Rabit(right)) => {
+            let left = left.metadata(None);
+            let right = right.metadata(None);
+            left.rotation_type == right.rotation_type
+                && left.code_dim == right.code_dim
+                && left.num_bits == right.num_bits
+                && left.packed == right.packed
+                && left.query_estimator == right.query_estimator
+                && left.fast_rotation_signs == right.fast_rotation_signs
+                && match (&left.rotate_mat, &right.rotate_mat) {
+                    (Some(left), Some(right)) => left.to_data() == right.to_data(),
+                    (None, None) => true,
+                    _ => false,
+                }
+        }
+        _ => false,
+    }
+}
+
 // TODO: move to `lance-index` crate.
 ///
 /// Returns (new_uuid, num_indices_merged, files)
@@ -407,6 +507,7 @@ pub(crate) async fn optimize_vector_indices(
     // try cast to v1 IVFIndex,
     // fallback to v2 IVFIndex if it's not v1 IVFIndex
     if !existing_indices[0].as_any().is::<IVFIndex>() {
+        validate_shared_vector_model(&existing_indices, "optimizing vector index")?;
         return optimize_vector_indices_v2(
             &dataset,
             unindexed,
@@ -1892,7 +1993,7 @@ pub(crate) async fn remap_index_file(
 
     let tasks = generate_remap_tasks(&index.ivf.offsets, &index.ivf.lengths)?;
 
-    let mut task_stream = stream::iter(tasks.into_iter())
+    let mut task_stream = stream::iter(tasks)
         .map(|task| task.load_and_remap(reader.clone(), index, mapping))
         .buffered(object_store.io_parallelism());
 
@@ -4267,19 +4368,26 @@ async fn train_streaming_coreset_ivf_model(
 
     let coreset_len = coreset.len();
     let (coreset_data, coreset_weights, coreset_losses) = coreset.into_fsl_parts(dimension)?;
-    let weighted_hierarchical_params = WeightedHierarchicalKMeansParams {
-        dimension,
-        target_k: num_partitions,
-        metric_type: DistanceType::L2,
-        max_iters: params.max_iters,
-        on_progress: on_progress.clone(),
+    // Scope `weighted_hierarchical_params` so the `on_progress` clone it holds
+    // (which owns a clone of `progress_tx`) is dropped as soon as training
+    // returns. Otherwise it would outlive the `progress_worker.await` below,
+    // keeping a channel sender alive so `progress_rx.recv()` never returns
+    // `None` and the progress worker — and thus this function — hangs forever.
+    let mut centroids = {
+        let weighted_hierarchical_params = WeightedHierarchicalKMeansParams {
+            dimension,
+            target_k: num_partitions,
+            metric_type: DistanceType::L2,
+            max_iters: params.max_iters,
+            on_progress: on_progress.clone(),
+        };
+        train_weighted_hierarchical_f32_kmeans(
+            &coreset_data,
+            &coreset_weights,
+            &coreset_losses,
+            &weighted_hierarchical_params,
+        )?
     };
-    let mut centroids = train_weighted_hierarchical_f32_kmeans(
-        &coreset_data,
-        &coreset_weights,
-        &coreset_losses,
-        &weighted_hierarchical_params,
-    )?;
     let refine_iters = 3;
     if refine_iters > 0 {
         let refined = refine_weighted_f32_kmeans(
@@ -4577,6 +4685,41 @@ mod tests {
 
     const DIM: usize = 32;
 
+    #[test]
+    fn test_shared_quantizer_model_compares_skipped_payloads() {
+        let codebook = |offset| {
+            let values = Float32Array::from_iter_values((0..32).map(|v| v as f32 + offset));
+            FixedSizeListArray::try_new_from_values(values, 2).unwrap()
+        };
+        let pq1 = Quantizer::Product(ProductQuantizer::new(
+            1,
+            4,
+            2,
+            codebook(0.0),
+            DistanceType::L2,
+        ));
+        let pq2 = Quantizer::Product(ProductQuantizer::new(
+            1,
+            4,
+            2,
+            codebook(100.0),
+            DistanceType::L2,
+        ));
+        assert!(!shared_quantizer_model(&pq1, &pq2));
+
+        let rq1 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::<Float32Type>(
+            1,
+            8,
+            lance_index::vector::bq::RQRotationType::Matrix,
+        ));
+        let rq2 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::<Float32Type>(
+            1,
+            8,
+            lance_index::vector::bq::RQRotationType::Matrix,
+        ));
+        assert!(!shared_quantizer_model(&rq1, &rq2));
+    }
+
     async fn compute_test_ivf_loss(dataset: &Dataset, column: &str, ivf: &IvfModel) -> f64 {
         let centroids = ivf
             .centroids_array()
@@ -5661,6 +5804,94 @@ mod tests {
         );
     }
 
+    /// Regression test for a hang in the streaming *coreset* trainer
+    /// (`train_streaming_coreset_ivf_model`, taken when `num_partitions > 256`).
+    ///
+    /// That function spawns a `progress_worker` task that loops on
+    /// `progress_rx.recv()` and only terminates once every clone of the mpsc
+    /// sender is dropped. The `on_progress` closure owns a sender clone, and
+    /// `WeightedHierarchicalKMeansParams` used to retain an `on_progress` clone
+    /// that outlived the `progress_worker.await` at the end of the function.
+    /// With a live sender remaining, `recv()` never returned `None`, the worker
+    /// never finished, and the trainer hung forever after all compute was done.
+    ///
+    /// The build is wrapped in a timeout so the regression fails fast rather
+    /// than hanging the test process indefinitely.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_streaming_coreset_ivf_training_terminates() {
+        use lance_index::progress::IndexBuildProgress;
+        use std::sync::atomic::{AtomicU64, Ordering};
+        use std::time::Duration;
+
+        #[derive(Debug, Default)]
+        struct CountingProgress {
+            progress_calls: AtomicU64,
+        }
+
+        #[async_trait::async_trait]
+        impl IndexBuildProgress for CountingProgress {
+            async fn stage_start(&self, _: &str, _: Option<u64>, _: &str) -> Result<()> {
+                Ok(())
+            }
+            async fn stage_progress(&self, _: &str, _: u64) -> Result<()> {
+                self.progress_calls.fetch_add(1, Ordering::Relaxed);
+                Ok(())
+            }
+            async fn stage_complete(&self, _: &str) -> Result<()> {
+                Ok(())
+            }
+        }
+
+        const SMALL_DIM: usize = 8;
+
+        let test_dir = TempStrDir::default();
+        let uri = format!("{}/ds", test_dir.as_str());
+        let reader = gen_batch()
+            .col("id", array::step::<UInt64Type>())
+            .col(
+                "vector",
+                array::rand_vec::<Float32Type>((SMALL_DIM as u32).into()),
+            )
+            .into_reader_rows(RowCount::from(2048), BatchCount::from(4));
+        let dataset = Dataset::write(reader, &uri, None).await.unwrap();
+
+        // > 256 partitions routes through `train_streaming_coreset_ivf_model`.
+        let mut params = IvfBuildParams::new(257);
+        params.sample_rate = 8;
+        params.streaming_sample_rate = Some(4);
+        params.streaming_refine_passes = 1;
+        params.max_iters = 2;
+
+        let progress = Arc::new(CountingProgress::default());
+
+        let ivf_model = tokio::time::timeout(
+            Duration::from_secs(120),
+            build_ivf_model(
+                &dataset,
+                "vector",
+                SMALL_DIM,
+                MetricType::L2,
+                &params,
+                None,
+                progress.clone(),
+            ),
+        )
+        .await
+        .expect(
+            "streaming coreset IVF training hung: progress worker never terminated after training",
+        )
+        .unwrap();
+
+        assert_eq!(ivf_model.num_partitions(), 257);
+        assert_eq!(ivf_model.dimension(), SMALL_DIM);
+        // The progress worker must have processed reports and then joined
+        // cleanly (proven by `build_ivf_model` returning at all).
+        assert!(
+            progress.progress_calls.load(Ordering::Relaxed) > 0,
+            "expected the progress worker to receive at least one report"
+        );
+    }
+
     #[test]
     fn test_fixed_training_ranges_are_sorted_and_bounded() {
         let ranges = generate_fixed_training_ranges(10_000, 1_234, 1_024, 16);
diff --git a/rust/lance/src/index/vector/ivf/io.rs b/rust/lance/src/index/vector/ivf/io.rs
index 612dd162281..a49398a0750 100644
--- a/rust/lance/src/index/vector/ivf/io.rs
+++ b/rust/lance/src/index/vector/ivf/io.rs
@@ -25,7 +25,6 @@ use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::previous::writer::FileWriter as PreviousFileWriter;
 use lance_index::metrics::NoOpMetricsCollector;
-use lance_index::scalar::IndexWriter;
 use lance_index::vector::hnsw::HNSW;
 use lance_index::vector::hnsw::{HnswMetadata, builder::HnswBuildParams};
 use lance_index::vector::ivf::storage::IvfModel;
@@ -44,6 +43,7 @@ use lance_table::format::SelfDescribingFileReader;
 use lance_table::io::manifest::ManifestDescribing;
 use object_store::path::Path;
 use tokio::sync::Semaphore;
+use tokio::task::JoinHandle;
 
 use crate::Result;
 
@@ -278,164 +278,234 @@ pub(super) async fn write_hnsw_quantization_index_partitions(
     }
 
     let object_store = ObjectStore::local();
-    let mut part_files = Vec::with_capacity(ivf.num_partitions());
-    let mut aux_part_files = Vec::with_capacity(ivf.num_partitions());
-    let tmp_part_dir = Path::from_filesystem_path(TempStdDir::default())?;
-    let mut tasks = Vec::with_capacity(ivf.num_partitions());
-    let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL));
-    for part_id in 0..ivf.num_partitions() {
-        part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id)));
-        aux_part_files.push(
-            tmp_part_dir
-                .clone()
-                .join(format!("hnsw_part_aux_{}", part_id)),
-        );
+    // Partitions are staged in this scratch dir, then merged into the final index.
+    // Share the guard with every task via `Arc` so its `Drop` removes the dir only
+    // after the last task finishes -- never while one is still writing.
+    let tmp_part_dir_guard = Arc::new(TempStdDir::default());
+    let tmp_part_dir = Path::from_filesystem_path(&**tmp_part_dir_guard)?;
+
+    // `Option` per handle so the consume loop can `take()` each one, leaving the
+    // not-yet-consumed handles for the error-path drain.
+    let mut tasks: Vec<Option<JoinHandle<Result<usize>>>> =
+        Vec::with_capacity(ivf.num_partitions());
+
+    let build_result: Result<(Vec<HnswMetadata>, IvfModel)> = async {
+        let mut part_files = Vec::with_capacity(ivf.num_partitions());
+        let mut aux_part_files = Vec::with_capacity(ivf.num_partitions());
+        let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL));
+        for part_id in 0..ivf.num_partitions() {
+            part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id)));
+            aux_part_files.push(
+                tmp_part_dir
+                    .clone()
+                    .join(format!("hnsw_part_aux_{}", part_id)),
+            );
 
-        let mut code_array: Vec<Arc<dyn Array>> = vec![];
-        let mut row_id_array: Vec<Arc<dyn Array>> = vec![];
+            let mut code_array: Vec<Arc<dyn Array>> = vec![];
+            let mut row_id_array: Vec<Arc<dyn Array>> = vec![];
 
-        // We don't transform vectors to SQ codes while shuffling,
-        // so we won't merge SQ codes from the stream.
+            // We don't transform vectors to SQ codes while shuffling,
+            // so we won't merge SQ codes from the stream.
 
-        if let Some(&previous_indices) = existing_indices.as_ref() {
-            for &idx in previous_indices.iter() {
-                let sub_index = idx
-                    .load_partition(part_id, true, &NoOpMetricsCollector)
-                    .await?;
-                let row_ids = Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned()));
-                row_id_array.push(row_ids);
+            if let Some(&previous_indices) = existing_indices.as_ref() {
+                for &idx in previous_indices.iter() {
+                    let sub_index = idx
+                        .load_partition(part_id, true, &NoOpMetricsCollector)
+                        .await?;
+                    let row_ids =
+                        Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned()));
+                    row_id_array.push(row_ids);
+                }
             }
-        }
-
-        let code_column = match &quantizer {
-            Quantizer::Product(pq) => Some(pq.column()),
-            _ => None,
-        };
-        merge_streams(
-            &mut streams_heap,
-            &mut new_streams,
-            part_id as u32,
-            code_column,
-            &mut code_array,
-            &mut row_id_array,
-        )
-        .await?;
 
-        if row_id_array.is_empty() {
-            tasks.push(tokio::spawn(async { Ok(0) }));
-            continue;
-        }
-
-        let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
-        let part_writer = PreviousFileWriter::<ManifestDescribing>::try_new(
-            &object_store,
-            part_file,
-            Schema::try_from(writer.schema())?,
-            &Default::default(),
-        )
-        .await?;
+            let code_column = match &quantizer {
+                Quantizer::Product(pq) => Some(pq.column()),
+                _ => None,
+            };
+            merge_streams(
+                &mut streams_heap,
+                &mut new_streams,
+                part_id as u32,
+                code_column,
+                &mut code_array,
+                &mut row_id_array,
+            )
+            .await?;
 
-        let aux_part_writer = match auxiliary_writer.as_ref() {
-            Some(writer) => Some(
-                PreviousFileWriter::<ManifestDescribing>::try_new(
-                    &object_store,
-                    aux_part_file,
-                    Schema::try_from(writer.schema())?,
-                    &Default::default(),
-                )
-                .await?,
-            ),
-            None => None,
-        };
+            if row_id_array.is_empty() {
+                tasks.push(Some(tokio::spawn(async { Ok(0) })));
+                continue;
+            }
 
-        let dataset = dataset.clone();
-        let column = column.to_owned();
-        let hnsw_params = hnsw_params.clone();
-        let quantizer = quantizer.clone();
-        let sem = sem.clone();
-        tasks.push(tokio::spawn(async move {
-            let _permit = sem.acquire().await.expect("semaphore error");
-
-            log::debug!("Building HNSW partition {}", part_id);
-            let result = build_hnsw_quantization_partition(
-                dataset,
-                &column,
-                distance_type,
-                hnsw_params,
-                part_writer,
-                aux_part_writer,
-                quantizer,
-                row_id_array,
-                code_array,
+            let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
+            let part_writer = PreviousFileWriter::<ManifestDescribing>::try_new(
+                &object_store,
+                part_file,
+                Schema::try_from(writer.schema())?,
+                &Default::default(),
             )
-            .await;
-            log::debug!("Finished building HNSW partition {}", part_id);
-            result
-        }));
-    }
-
-    let mut aux_ivf = IvfModel::empty();
-    let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions());
-    for (part_id, task) in tasks.into_iter().enumerate() {
-        let offset = writer.len();
-        let num_rows = task.await??;
+            .await?;
 
-        if num_rows == 0 {
-            ivf.add_partition(0);
-            aux_ivf.add_partition(0);
-            hnsw_metadata.push(HnswMetadata::default());
-            continue;
+            let aux_part_writer = match auxiliary_writer.as_ref() {
+                Some(writer) => Some(
+                    PreviousFileWriter::<ManifestDescribing>::try_new(
+                        &object_store,
+                        aux_part_file,
+                        Schema::try_from(writer.schema())?,
+                        &Default::default(),
+                    )
+                    .await?,
+                ),
+                None => None,
+            };
+
+            let dataset = dataset.clone();
+            let column = column.to_owned();
+            let hnsw_params = hnsw_params.clone();
+            let quantizer = quantizer.clone();
+            let sem = sem.clone();
+            let tmp_part_dir_guard = tmp_part_dir_guard.clone();
+            tasks.push(Some(tokio::spawn(async move {
+                // Hold a guard clone so the scratch dir stays alive while this task writes.
+                let _tmp_part_dir_guard = tmp_part_dir_guard;
+                let _permit = sem.acquire().await.map_err(|err| {
+                    Error::io(format!(
+                        "failed to acquire HNSW partition build permit: {err}"
+                    ))
+                })?;
+
+                log::debug!("Building HNSW partition {}", part_id);
+                let result = build_hnsw_quantization_partition(
+                    dataset,
+                    &column,
+                    distance_type,
+                    hnsw_params,
+                    part_writer,
+                    aux_part_writer,
+                    quantizer,
+                    row_id_array,
+                    code_array,
+                )
+                .await;
+                log::debug!("Finished building HNSW partition {}", part_id);
+                result
+            })));
         }
 
-        let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
-        let part_reader =
-            PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?;
+        let mut aux_ivf = IvfModel::empty();
+        let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions());
+        for (part_id, task) in tasks.iter_mut().enumerate() {
+            let task = task
+                .take()
+                .expect("each partition task is consumed exactly once");
+            let offset = writer.len();
+            let num_rows = task.await??;
+
+            if num_rows == 0 {
+                ivf.add_partition(0);
+                aux_ivf.add_partition(0);
+                hnsw_metadata.push(HnswMetadata::default());
+                continue;
+            }
 
-        let batches = futures::stream::iter(0..part_reader.num_batches())
-            .map(|batch_id| {
-                part_reader.read_batch(
-                    batch_id as i32,
-                    ReadBatchParams::RangeFull,
-                    part_reader.schema(),
-                )
-            })
-            .buffered(object_store.io_parallelism())
-            .try_collect::<Vec<_>>()
-            .await?;
-        writer.write(&batches).await?;
-
-        ivf.add_partition((writer.len() - offset) as u32);
-        hnsw_metadata.push(serde_json::from_str(
-            part_reader.schema().metadata[HNSW::metadata_key()].as_str(),
-        )?);
-        std::mem::drop(part_reader);
-        object_store.delete(part_file).await?;
-
-        if let Some(aux_writer) = auxiliary_writer.as_mut() {
-            let aux_part_reader =
-                PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None)
-                    .await?;
+            let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
+            let part_reader =
+                PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?;
 
-            let batches = futures::stream::iter(0..aux_part_reader.num_batches())
+            let batches = futures::stream::iter(0..part_reader.num_batches())
                 .map(|batch_id| {
-                    aux_part_reader.read_batch(
+                    part_reader.read_batch(
                         batch_id as i32,
                         ReadBatchParams::RangeFull,
-                        aux_part_reader.schema(),
+                        part_reader.schema(),
                     )
                 })
                 .buffered(object_store.io_parallelism())
                 .try_collect::<Vec<_>>()
                 .await?;
-            std::mem::drop(aux_part_reader);
-            object_store.delete(aux_part_file).await?;
+            writer.write(&batches).await?;
+
+            ivf.add_partition((writer.len() - offset) as u32);
+            hnsw_metadata.push(serde_json::from_str(
+                part_reader.schema().metadata[HNSW::metadata_key()].as_str(),
+            )?);
+            std::mem::drop(part_reader);
+            object_store.delete(part_file).await?;
+
+            if let Some(aux_writer) = auxiliary_writer.as_mut() {
+                let aux_part_reader =
+                    PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None)
+                        .await?;
+
+                let batches = futures::stream::iter(0..aux_part_reader.num_batches())
+                    .map(|batch_id| {
+                        aux_part_reader.read_batch(
+                            batch_id as i32,
+                            ReadBatchParams::RangeFull,
+                            aux_part_reader.schema(),
+                        )
+                    })
+                    .buffered(object_store.io_parallelism())
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                std::mem::drop(aux_part_reader);
+                object_store.delete(aux_part_file).await?;
+
+                aux_writer.write(&batches).await?;
+                aux_ivf.add_partition(num_rows as u32);
+            }
+        }
+
+        Ok((hnsw_metadata, aux_ivf))
+    }
+    .await;
+
+    // On error, abort and await the partition builds we never consumed so none
+    // keep running in the background; see `drain_partition_tasks`.
+    if build_result.is_err() {
+        for err in drain_partition_tasks(&mut tasks).await {
+            log::warn!(
+                "HNSW partition build task failed while draining after an earlier error: {err}"
+            );
+        }
+    }
+
+    build_result
+}
 
-            aux_writer.write(&batches).await?;
-            aux_ivf.add_partition(num_rows as u32);
+/// Abort and await every still-outstanding partition-build task, returning the
+/// non-cancellation errors they surfaced.
+///
+/// A dropped [`JoinHandle`] detaches its task, so every handle is aborted up front
+/// before any is awaited: otherwise a task slow to observe its own cancellation
+/// would keep running -- and keep writing into the scratch dir -- while an earlier
+/// handle is still being awaited. Awaiting then resolves only once each task has
+/// actually stopped. Cancellation errors are the expected result of the abort and
+/// dropped; failures and panics from tasks that had already finished before the
+/// abort are returned so the caller can surface them (a task still in flight is
+/// cancelled, so this best-effort drain only reports errors already produced).
+async fn drain_partition_tasks(tasks: &mut [Option<JoinHandle<Result<usize>>>]) -> Vec<Error> {
+    for task in tasks.iter() {
+        if let Some(handle) = task.as_ref() {
+            handle.abort();
         }
     }
 
-    Ok((hnsw_metadata, aux_ivf))
+    let mut errors = Vec::with_capacity(tasks.len());
+    for task in tasks.iter_mut() {
+        let Some(handle) = task.take() else {
+            continue;
+        };
+        match handle.await {
+            Ok(Ok(_)) => {}
+            Ok(Err(e)) => errors.push(e),
+            Err(join_err) if join_err.is_cancelled() => {}
+            Err(join_err) => errors.push(Error::io(format!(
+                "HNSW partition build task panicked: {join_err}"
+            ))),
+        }
+    }
+    errors
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -473,24 +543,29 @@ async fn build_hnsw_quantization_partition(
     let build_hnsw =
         build_and_write_hnsw(vectors.clone(), (*hnsw_params).clone(), metric_type, writer);
 
+    // Build PQ storage as a child future, joined below: it writes `aux_writer`'s
+    // file into the scratch dir, so it is cancelled together with this task when the
+    // error-path drain aborts it, and the join surfaces its errors.
     let build_store = match quantizer {
         Quantizer::Flat(_) => {
             return Err(Error::index(
                 "Flat quantizer is not supported for IVF_HNSW".to_string(),
             ));
         }
-        Quantizer::Product(pq) => tokio::spawn(build_and_write_pq_storage(
-            metric_type,
-            row_ids,
-            code_array,
-            pq,
-            aux_writer.unwrap(),
-        )),
-
-        _ => unreachable!("IVF_HNSW_SQ has been moved to v2 index builder"),
+        Quantizer::Product(pq) => {
+            let aux_writer = aux_writer.ok_or_else(|| {
+                Error::index("IVF_HNSW_PQ requires an auxiliary writer for PQ storage".to_string())
+            })?;
+            build_and_write_pq_storage(metric_type, row_ids, code_array, pq, aux_writer)
+        }
+        _ => {
+            return Err(Error::index(
+                "IVF_HNSW_SQ is not supported in the legacy HNSW partition writer".to_string(),
+            ));
+        }
     };
 
-    let index_rows = futures::join!(build_hnsw, build_store).0?;
+    let (index_rows, ()) = futures::try_join!(build_hnsw, build_store)?;
     assert!(
         index_rows >= num_rows,
         "index rows {} must be greater than or equal to num rows {}",
@@ -508,7 +583,7 @@ async fn build_and_write_hnsw(
 ) -> Result<usize> {
     let batch = params.build(vectors, distance_type).await?.to_batch()?;
     let metadata = batch.schema_ref().metadata().clone();
-    writer.write_record_batch(batch).await?;
+    writer.write(&[batch]).await?;
     Ok(writer.finish_with_metadata(&metadata).await?.num_rows as usize)
 }
 
@@ -521,7 +596,7 @@ async fn build_and_write_pq_storage(
 ) -> Result<()> {
     let storage = spawn_cpu(move || build_pq_storage(metric_type, row_ids, code_array, pq)).await?;
 
-    writer.write_record_batch(storage.batch().clone()).await?;
+    writer.write(&[storage.batch().clone()]).await?;
     writer.finish().await?;
     Ok(())
 }
@@ -530,6 +605,9 @@ async fn build_and_write_pq_storage(
 mod tests {
     use super::*;
 
+    use std::path::PathBuf;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
     use crate::Dataset;
     use crate::index::vector::ivf::v2;
     use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams};
@@ -538,6 +616,8 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_index::IndexType;
     use lance_index::metrics::NoOpMetricsCollector;
+    use lance_index::vector::ivf::IvfBuildParams;
+    use lance_index::vector::pq::PQBuildParams;
     use lance_testing::datagen::generate_random_array;
 
     #[tokio::test]
@@ -589,4 +669,248 @@ mod tests {
 
         //let indices = /ds.
     }
+
+    /// The scratch dir must outlive every partition task: dropping the caller-side
+    /// guard while tasks are still in flight must not remove it, because each task
+    /// still holds an `Arc` clone of the guard.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_scratch_dir_outlives_partition_tasks() {
+        let tmp_part_dir_guard = Arc::new(TempStdDir::default());
+        let scratch_path = tmp_part_dir_guard.to_path_buf();
+
+        // Park each task until the caller-side guard is dropped, so the dir's
+        // survival is attributable solely to the clones the tasks still hold.
+        let running = Arc::new(AtomicUsize::new(0));
+        let released = Arc::new(AtomicBool::new(false));
+
+        const NUM_TASKS: usize = 3;
+        let mut tasks = Vec::with_capacity(NUM_TASKS);
+        for _ in 0..NUM_TASKS {
+            let task_guard = tmp_part_dir_guard.clone();
+            let running = running.clone();
+            let released = released.clone();
+            tasks.push(tokio::spawn(async move {
+                running.fetch_add(1, Ordering::SeqCst);
+                while !released.load(Ordering::SeqCst) {
+                    tokio::task::yield_now().await;
+                }
+                // Still holding `task_guard`, so the directory must be live.
+                task_guard.exists()
+            }));
+        }
+
+        // Drop the caller-side guard only once every task is parked holding its clone.
+        while running.load(Ordering::SeqCst) < NUM_TASKS {
+            tokio::task::yield_now().await;
+        }
+        drop(tmp_part_dir_guard);
+        assert!(
+            scratch_path.exists(),
+            "scratch dir removed while partition tasks still held the guard"
+        );
+
+        released.store(true, Ordering::SeqCst);
+        for task in tasks {
+            assert!(
+                task.await.unwrap(),
+                "a partition task observed its scratch dir already removed"
+            );
+        }
+        assert!(
+            !scratch_path.exists(),
+            "scratch dir was not removed after the last task's guard clone dropped"
+        );
+    }
+
+    /// The drain step must outlive every spawned task: it aborts and awaits each
+    /// one so that, once it returns, no task is still running against the scratch
+    /// directory. It must also surface late failures rather than swallow them.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_drain_partition_tasks_waits_and_reports_errors() {
+        // Scratch dir guard analogous to the one held by
+        // `write_hnsw_quantization_index_partitions`; tasks read from it, and it
+        // is dropped only after the drain completes.
+        let scratch_guard = TempStdDir::default();
+        let scratch_path = scratch_guard.to_path_buf();
+
+        // Count of live task futures. `LiveGuard` decrements on both completion and
+        // cancellation, so a zero count after the drain proves every task terminated
+        // rather than being detached.
+        let live = Arc::new(AtomicUsize::new(0));
+        let saw_missing_dir = Arc::new(AtomicBool::new(false));
+
+        struct LiveGuard(Arc<AtomicUsize>);
+        impl Drop for LiveGuard {
+            fn drop(&mut self) {
+                self.0.fetch_sub(1, Ordering::SeqCst);
+            }
+        }
+
+        const NUM_SLOW: usize = 3;
+        let mut tasks: Vec<Option<JoinHandle<Result<usize>>>> = Vec::with_capacity(NUM_SLOW + 1);
+
+        // Tasks that never resolve on their own: the drain's abort is the only thing
+        // that can stop them, which is exactly what this test exercises.
+        for _ in 0..NUM_SLOW {
+            let live = live.clone();
+            let saw_missing_dir = saw_missing_dir.clone();
+            let scratch_path = scratch_path.clone();
+            tasks.push(Some(tokio::spawn(async move {
+                live.fetch_add(1, Ordering::SeqCst);
+                let _guard = LiveGuard(live.clone());
+                if !scratch_path.exists() {
+                    saw_missing_dir.store(true, Ordering::SeqCst);
+                }
+                futures::future::pending::<()>().await;
+                Ok(0)
+            })));
+        }
+
+        // A task that fails; its error must be returned, not silently dropped.
+        // The drain aborts every handle up front, and an abort only preserves a
+        // task's output if the task has already finished -- an in-flight task is
+        // cancelled and its error lost. So wait until this task has actually
+        // completed before handing it to the drain; otherwise whether its error
+        // surfaces would depend on the scheduler and the test would be flaky.
+        let failing =
+            tokio::spawn(async move { Err(Error::io("late partition failure".to_string())) });
+        while !failing.is_finished() {
+            tokio::task::yield_now().await;
+        }
+        tasks.push(Some(failing));
+
+        // Ensure all slow tasks are actually running before draining, so the
+        // drain has to await their cancellation rather than aborting them before
+        // they ever start.
+        while live.load(Ordering::SeqCst) < NUM_SLOW {
+            tokio::task::yield_now().await;
+        }
+
+        let errors = drain_partition_tasks(&mut tasks).await;
+
+        assert!(tasks.iter().all(Option::is_none), "handles left undrained");
+        assert_eq!(
+            live.load(Ordering::SeqCst),
+            0,
+            "a task was still running after the drain returned"
+        );
+        assert!(
+            !saw_missing_dir.load(Ordering::SeqCst),
+            "scratch dir was removed while a task was still running"
+        );
+        assert_eq!(errors.len(), 1, "expected exactly the one late failure");
+        assert!(
+            errors[0].to_string().contains("late partition failure"),
+            "late failure was not surfaced: {}",
+            errors[0]
+        );
+
+        // The guard, not the drain, removes the scratch dir.
+        assert!(scratch_path.exists());
+        drop(scratch_guard);
+        assert!(!scratch_path.exists());
+    }
+
+    /// `write_hnsw_quantization_index_partitions` stages each partition in a scratch
+    /// directory owned by a [`TempStdDir`] guard, which must remove it once the build
+    /// finishes so the OS temp dir does not grow without bound across legacy
+    /// IVF_HNSW_* builds.
+    ///
+    /// The OS temp dir is process-global, so an in-process check can't attribute a
+    /// leak to our own build. Instead we run the build in a child process with
+    /// `TMPDIR` pointed at an isolated dir we own, then assert nothing survives.
+    #[test]
+    fn test_hnsw_pq_scratch_dir_is_not_leaked() {
+        // Isolated temp root for the child. Owned here so it -- and anything the
+        // child leaks into it -- is removed when this guard drops at test end.
+        let isolated_root = TempStdDir::default();
+
+        let child_test = "index::vector::ivf::io::tests::build_legacy_hnsw_pq_in_child_process";
+        let output = std::process::Command::new(std::env::current_exe().unwrap())
+            .args([child_test, "--exact", "--ignored", "--nocapture"])
+            .env("TMPDIR", isolated_root.as_ref())
+            .env("LANCE_HNSW_LEAK_TEST_ROOT", isolated_root.as_ref())
+            .output()
+            .expect("failed to spawn child test process");
+        assert!(
+            output.status.success(),
+            "child build process failed:\n--- stdout ---\n{}\n--- stderr ---\n{}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        );
+
+        // Each build stages its partitions in one `.tmp*` dir directly under TMPDIR.
+        // Every guard removes its dir when it drops, so none should survive.
+        let leaked: Vec<PathBuf> = std::fs::read_dir(&isolated_root)
+            .expect("read isolated temp root")
+            .flatten()
+            .map(|entry| entry.path())
+            .filter(|path| {
+                path.is_dir()
+                    && path
+                        .file_name()
+                        .and_then(|name| name.to_str())
+                        .is_some_and(|name| name.starts_with(".tmp"))
+            })
+            .collect();
+
+        assert!(
+            leaked.is_empty(),
+            "legacy IVF_HNSW_PQ build leaked {} scratch director{} under the temp dir; \
+             the TempStdDir guard should remove each one when it drops: {:?}",
+            leaked.len(),
+            if leaked.len() == 1 { "y" } else { "ies" },
+            leaked,
+        );
+    }
+
+    /// Child half of [`test_hnsw_pq_scratch_dir_is_not_leaked`]. Ignored so it only
+    /// runs when the parent spawns it with `TMPDIR` and `LANCE_HNSW_LEAK_TEST_ROOT`
+    /// pointed at an isolated dir. Builds a few legacy IVF_HNSW_PQ indices; the
+    /// parent does the leak detection.
+    #[tokio::test]
+    #[ignore = "spawned as a child process by test_hnsw_pq_scratch_dir_is_not_leaked"]
+    async fn build_legacy_hnsw_pq_in_child_process() {
+        // Only do work when spawned by the parent; a bare `--ignored` run leaves
+        // the variable unset, so no-op rather than fail.
+        let Ok(root) = std::env::var("LANCE_HNSW_LEAK_TEST_ROOT") else {
+            return;
+        };
+
+        const DIM: usize = 32;
+        const ROWS: usize = 1024;
+        const NLIST: usize = 4;
+        const NUM_BUILDS: usize = 3;
+
+        // Keep the dataset out of the temp dir's `.tmp*` namespace so the parent
+        // never confuses it with a leaked scratch directory.
+        let dataset_uri = format!("{root}/dataset");
+        let values = generate_random_array(ROWS * DIM);
+        let fsl = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap());
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "vector",
+            fsl.data_type().clone(),
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![fsl]).unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let ds = Dataset::write(batches, &dataset_uri, Default::default())
+            .await
+            .unwrap();
+
+        for _ in 0..NUM_BUILDS {
+            crate::index::vector::ivf::build_ivf_hnsw_pq_index(
+                &ds,
+                "vector",
+                "idx",
+                uuid::Uuid::new_v4(),
+                MetricType::L2,
+                &IvfBuildParams::new(NLIST),
+                &HnswBuildParams::default(),
+                &PQBuildParams::new(4, 8),
+            )
+            .await
+            .unwrap();
+        }
+    }
 }
diff --git a/rust/lance/src/index/vector/ivf/v2.rs b/rust/lance/src/index/vector/ivf/v2.rs
index 0faa79dce5f..2f18dbe8889 100644
--- a/rust/lance/src/index/vector/ivf/v2.rs
+++ b/rust/lance/src/index/vector/ivf/v2.rs
@@ -9,7 +9,7 @@ use std::{
     any::Any,
     borrow::Cow,
     collections::{BinaryHeap, HashMap},
-    sync::{Arc, Mutex},
+    sync::{Arc, LazyLock, Mutex},
 };
 
 use crate::index::vector::{IndexFileVersion, builder::index_type_string};
@@ -122,6 +122,41 @@ pub(crate) struct IvfIndexState<Q: Quantization> {
     pub(crate) rq_search_cache: RabitSearchCacheCell,
 }
 
+/// Number of prepared partitions handed to a single `spawn_cpu` dispatch on the
+/// streaming search path.
+///
+/// The streaming path deliberately avoids per-partition CPU-task fan-out (a measured
+/// 14-30% latency win, see #6475). Searching a batch of partitions per `spawn_cpu`
+/// keeps most of that benefit — the per-dispatch overhead is paid once per
+/// `STREAMING_SEARCH_BATCH_SIZE` partitions instead of once per partition — while
+/// keeping the channel `recv`/`send` in async code so no CPU-pool thread ever parks on
+/// a channel (which can deadlock the pool on small hosts, see #7642). `should_stop` is
+/// still checked per partition, so early-stop granularity is unchanged.
+///
+/// This is a tunable knob: larger batches amortize dispatch overhead further and keep
+/// more work on a single CPU thread, at the cost of more prepared partitions held in
+/// memory at once. The batch is an upper bound: the search loop greedily drains
+/// whatever is already prepared rather than waiting for a full batch, so a slow
+/// producer yields small batches (matching the old search-as-it-arrives latency) and
+/// only a fast producer fills whole ones. Override with the
+/// `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE` environment variable.
+pub(crate) const DEFAULT_STREAMING_SEARCH_BATCH_SIZE: usize = 16;
+
+pub(crate) static STREAMING_SEARCH_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
+    let batch_size = std::env::var("LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE")
+        .map(|value| {
+            value
+                .parse()
+                .expect("failed to parse LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE")
+        })
+        .unwrap_or(DEFAULT_STREAMING_SEARCH_BATCH_SIZE);
+    assert!(
+        batch_size > 0,
+        "LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE must be greater than 0, got {batch_size}"
+    );
+    batch_size
+});
+
 struct PreparedPartitionSearch<S: IvfSubIndex, Q: Quantization> {
     query: Query,
     pre_filter: Arc<dyn PreFilter>,
@@ -1622,8 +1657,11 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
             )));
         }
 
+        // The prepared channel holds a full search batch so that partitions prepared
+        // while the previous batch is being searched are ready for the next greedy
+        // drain, instead of serializing producer and consumer through a single slot.
         let (prepared_tx, mut prepared_rx) =
-            mpsc::channel::<Result<PreparedPartitionSearch<S, Q>>>(1);
+            mpsc::channel::<Result<PreparedPartitionSearch<S, Q>>>(*STREAMING_SEARCH_BATCH_SIZE);
         let (batch_tx, batch_rx) = mpsc::channel::<DataFusionResult<RecordBatch>>(1);
 
         let prepare_index = self.clone();
@@ -1665,61 +1703,139 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         let use_query_residual = self.use_query_residual;
         let use_residual_scratch = self.use_residual_scratch;
         let search_metrics = metrics.clone();
-        let batch_tx_for_search = batch_tx.clone();
         let search_control = control.clone();
         let scratch_pool = self.scratch_pool.clone();
+        // Search prepared partitions in batches. Each batch is searched in a single
+        // `spawn_cpu` dispatch (amortizing the per-dispatch overhead the single-worker
+        // design in #6475 avoided), but the channel `recv`/`send` stay in async code so
+        // no CPU-pool thread ever parks on a channel — parking one can deadlock the pool
+        // on small hosts (#7642). `should_stop` is checked per partition, so early-stop
+        // granularity is unchanged.
+        //
+        // Batches are formed greedily: wait for one prepared partition, then drain
+        // whatever else is already prepared, up to the batch size. Waiting for a full
+        // batch instead would delay the first search (and the early-stop feedback it
+        // produces) behind up to a whole batch of prepare I/O, which is significant
+        // when prepare parallelism is low.
         tokio::spawn(async move {
-            let search_result = spawn_cpu(move || -> DataFusionResult<()> {
-                scratch_pool.with_scratch(|scratch| {
-                    while let Some(prepared) = prepared_rx.blocking_recv() {
-                        let prepared = match prepared {
-                            Ok(prepared) => prepared,
-                            Err(err) => {
-                                let _ = batch_tx_for_search
-                                    .blocking_send(Err(DataFusionError::from(err)));
-                                return Ok(());
-                            }
-                        };
+            loop {
+                // Stop pulling as soon as the search is done — or the receiver of our
+                // results is gone — so the producer stops preparing partitions we
+                // would never search.
+                if search_control
+                    .as_ref()
+                    .is_some_and(|control| control.should_stop())
+                    || batch_tx.is_closed()
+                {
+                    return;
+                }
 
-                        if search_control
-                            .as_ref()
-                            .is_some_and(|control| control.should_stop())
-                        {
-                            return Ok(());
+                let mut prepared_batch = Vec::with_capacity(*STREAMING_SEARCH_BATCH_SIZE);
+                let mut prepare_error = None;
+                let mut producer_done = false;
+                match prepared_rx.recv().await {
+                    Some(Ok(prepared)) => prepared_batch.push(prepared),
+                    Some(Err(err)) => prepare_error = Some(DataFusionError::from(err)),
+                    None => producer_done = true,
+                }
+                while prepare_error.is_none()
+                    && !producer_done
+                    && prepared_batch.len() < *STREAMING_SEARCH_BATCH_SIZE
+                {
+                    match prepared_rx.try_recv() {
+                        Ok(Ok(prepared)) => prepared_batch.push(prepared),
+                        Ok(Err(err)) => {
+                            prepare_error = Some(DataFusionError::from(err));
                         }
-
-                        let batch = {
-                            Self::run_prepared_partition_search(
-                                use_query_residual,
-                                use_residual_scratch,
-                                prepared,
-                                search_metrics.as_ref(),
-                                scratch,
-                            )
+                        // Nothing else is prepared yet; search what we have rather
+                        // than waiting for more.
+                        Err(mpsc::error::TryRecvError::Empty) => break,
+                        Err(mpsc::error::TryRecvError::Disconnected) => {
+                            producer_done = true;
                         }
-                        .map_err(DataFusionError::from);
-                        match batch {
-                            Ok(batch) => {
-                                if let Some(control) = search_control.as_ref() {
-                                    control.record_batch(&batch);
+                    }
+                }
+
+                if !prepared_batch.is_empty() {
+                    let scratch_pool = scratch_pool.clone();
+                    let search_metrics = search_metrics.clone();
+                    let search_control = search_control.clone();
+                    // `is_closed` is synchronously callable, so a sender clone lets the
+                    // CPU loop notice a dropped receiver between partitions instead of
+                    // searching out the whole batch for a cancelled query. (A `select!`
+                    // on `closed()` would not help here: `spawn_cpu` closures are not
+                    // cancellable, so abandoning the await leaves the work running.)
+                    let cancel_probe = batch_tx.clone();
+                    let search_output = spawn_cpu(move || {
+                        let mut outputs: Vec<DataFusionResult<RecordBatch>> =
+                            Vec::with_capacity(prepared_batch.len());
+                        // `stopped` means the whole search should end (an error, an
+                        // early-stop signal, or cancellation), not just this batch.
+                        let mut stopped = false;
+                        scratch_pool.with_scratch(|scratch| {
+                            for prepared in prepared_batch {
+                                if search_control
+                                    .as_ref()
+                                    .is_some_and(|control| control.should_stop())
+                                    || cancel_probe.is_closed()
+                                {
+                                    stopped = true;
+                                    break;
                                 }
-                                if batch_tx_for_search.blocking_send(Ok(batch)).is_err() {
-                                    return Ok(());
+                                match Self::run_prepared_partition_search(
+                                    use_query_residual,
+                                    use_residual_scratch,
+                                    prepared,
+                                    search_metrics.as_ref(),
+                                    scratch,
+                                )
+                                .map_err(DataFusionError::from)
+                                {
+                                    Ok(batch) => {
+                                        if let Some(control) = search_control.as_ref() {
+                                            control.record_batch(&batch);
+                                        }
+                                        outputs.push(Ok(batch));
+                                    }
+                                    Err(err) => {
+                                        outputs.push(Err(err));
+                                        stopped = true;
+                                        break;
+                                    }
                                 }
                             }
-                            Err(err) => {
-                                let _ = batch_tx_for_search.blocking_send(Err(err));
-                                return Ok(());
-                            }
+                        });
+                        Ok::<_, DataFusionError>((outputs, stopped))
+                    })
+                    .await;
+
+                    let (outputs, stopped) = match search_output {
+                        Ok(output) => output,
+                        // Defensive: the closure always returns Ok (search errors are
+                        // captured per partition in `outputs`), so this arm should be
+                        // unreachable. Forward and stop rather than drop silently.
+                        Err(err) => {
+                            let _ = batch_tx.send(Err(err)).await;
+                            return;
+                        }
+                    };
+                    for output in outputs {
+                        if batch_tx.send(output).await.is_err() {
+                            return;
                         }
                     }
-                    Ok(())
-                })
-            })
-            .await;
+                    if stopped {
+                        return;
+                    }
+                }
 
-            if let Err(err) = search_result {
-                let _ = batch_tx.send(Err(err)).await;
+                if let Some(err) = prepare_error {
+                    let _ = batch_tx.send(Err(err)).await;
+                    return;
+                }
+                if producer_done {
+                    return;
+                }
             }
         });
 
@@ -1729,6 +1845,147 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         )))
     }
 
+    fn supports_batch_partition_search(&self) -> bool {
+        S::supports_global_topk_heap()
+    }
+
+    async fn search_partitions_batch(
+        self: Arc<Self>,
+        query: Query,
+        partitions_per_query: Vec<Arc<UInt32Array>>,
+        q_c_dists_per_query: Vec<Arc<Float32Array>>,
+        pre_filter: Arc<dyn PreFilter>,
+        metrics: Arc<dyn MetricsCollector>,
+    ) -> Result<Vec<RecordBatch>> {
+        if !S::supports_global_topk_heap() {
+            return Err(Error::not_supported(
+                "batch partition search requires a global top-k heap sub-index",
+            ));
+        }
+        let query_count = partitions_per_query.len();
+        if q_c_dists_per_query.len() != query_count {
+            return Err(Error::invalid_input(format!(
+                "batch partition search: {query_count} query partition lists but {} distance lists",
+                q_c_dists_per_query.len()
+            )));
+        }
+        if query_count == 0 {
+            return Ok(Vec::new());
+        }
+        if !query.key.len().is_multiple_of(query_count) {
+            return Err(Error::invalid_input(format!(
+                "batch partition search: query key length {} is not divisible by query count {query_count}",
+                query.key.len()
+            )));
+        }
+        let dim = query.key.len() / query_count;
+
+        // Per-query immutable search state: the query vector slice and the
+        // optional Rabit raw-query context both depend only on the query vector,
+        // so compute them once up front rather than per probed partition.
+        let mut base_queries = Vec::with_capacity(query_count);
+        let mut raw_query_contexts = Vec::with_capacity(query_count);
+        for query_index in 0..query_count {
+            if partitions_per_query[query_index].len() != q_c_dists_per_query[query_index].len() {
+                return Err(Error::invalid_input(format!(
+                    "batch partition search: query {query_index} has {} partitions but {} distances",
+                    partitions_per_query[query_index].len(),
+                    q_c_dists_per_query[query_index].len()
+                )));
+            }
+            let mut single_query = query.clone();
+            single_query.key = query.key.slice(query_index * dim, dim);
+            raw_query_contexts.push(self.prepare_rq_raw_query_context(&single_query.key)?);
+            base_queries.push(single_query);
+        }
+
+        // Invert the per-query partition lists so each distinct partition is
+        // loaded once and scored against every query that probes it.
+        let mut assignments: HashMap<u32, Vec<(usize, f32)>> = HashMap::new();
+        for (query_index, (parts, dists)) in partitions_per_query
+            .iter()
+            .zip(q_c_dists_per_query.iter())
+            .enumerate()
+        {
+            for (part_id, dist_q_c) in parts.values().iter().zip(dists.values().iter()) {
+                assignments
+                    .entry(*part_id)
+                    .or_default()
+                    .push((query_index, *dist_q_c));
+            }
+        }
+
+        pre_filter.wait_for_ready().await?;
+
+        // Load each distinct partition's storage exactly once. This shared I/O
+        // is the whole point of batch search versus repeated single queries.
+        let load_parallelism = get_num_compute_intensive_cpus().max(1);
+        let load_index = self.clone();
+        let load_metrics = metrics.clone();
+        let loaded = stream::iter(assignments)
+            .map(move |(part_id, probing_queries)| {
+                let index = load_index.clone();
+                let metrics = load_metrics.clone();
+                async move {
+                    let part_entry = index
+                        .load_partition(part_id as usize, true, metrics.as_ref())
+                        .await?;
+                    Result::Ok((part_id as usize, part_entry, probing_queries))
+                }
+            })
+            .buffered(load_parallelism)
+            .try_collect::<Vec<_>>()
+            .await?;
+
+        // Score the loaded partitions into one top-k heap per query.
+        let use_query_residual = self.use_query_residual;
+        let use_residual_scratch = self.use_residual_scratch;
+        let heap_capacity = query.k * query.refine_factor.unwrap_or(1) as usize;
+        let scratch_pool = self.scratch_pool.clone();
+        let index = self.clone();
+        let search_metrics = metrics.clone();
+        let batches = spawn_cpu(move || -> Result<Vec<RecordBatch>> {
+            let mut heaps: Vec<BinaryHeap<OrderedNode<u64>>> = (0..query_count)
+                .map(|_| BinaryHeap::with_capacity(heap_capacity))
+                .collect();
+            scratch_pool.with_scratch(|scratch| -> Result<()> {
+                for (part_id, part_entry, probing_queries) in &loaded {
+                    let partition_centroid = index.ivf.centroid(*part_id);
+                    for (query_index, dist_q_c) in probing_queries {
+                        let mut single_query = base_queries[*query_index].clone();
+                        single_query.dist_q_c = *dist_q_c;
+                        let prepared = PreparedPartitionSearch::<S, Q> {
+                            query: single_query,
+                            pre_filter: pre_filter.clone(),
+                            partition_id: *part_id,
+                            partition_centroid: partition_centroid.clone(),
+                            rq_search_cache: index.rq_search_cache.clone(),
+                            raw_query_context: raw_query_contexts[*query_index].clone(),
+                            part_entry: part_entry.clone(),
+                            _marker: PhantomData,
+                        };
+                        Self::accumulate_prepared_partition_search(
+                            use_query_residual,
+                            use_residual_scratch,
+                            prepared,
+                            &mut heaps[*query_index],
+                            scratch,
+                            search_metrics.as_ref(),
+                        )?;
+                    }
+                }
+                Ok(())
+            })?;
+            heaps
+                .into_iter()
+                .map(Self::global_heap_to_batch)
+                .collect::<Result<Vec<_>>>()
+        })
+        .await?;
+
+        Ok(batches)
+    }
+
     fn is_loadable(&self) -> bool {
         false
     }
@@ -1939,7 +2196,7 @@ mod tests {
     use crate::dataset::{InsertBuilder, UpdateBuilder, WriteMode, WriteParams};
     use crate::index::DatasetIndexExt;
     use crate::index::DatasetIndexInternalExt;
-    use crate::index::vector::ivf::v2::IvfPq;
+    use crate::index::vector::ivf::v2::{IvfFlatIndex, IvfPq};
     use crate::utils::test::copy_test_data_to_tmp;
     use crate::{
         Dataset,
@@ -1956,6 +2213,7 @@ mod tests {
     use lance_file::reader::{FileReader, FileReaderOptions};
     use lance_file::writer::FileWriter;
     use lance_index::IndexType;
+    use lance_index::optimize::OptimizeOptions;
     use lance_index::progress::IndexBuildProgress;
     use lance_index::vector::DIST_COL;
     use lance_index::vector::hnsw::builder::HnswBuildParams;
@@ -1970,7 +2228,6 @@ mod tests {
         storage::STORAGE_METADATA_KEY,
     };
     use lance_index::{INDEX_AUXILIARY_FILE_NAME, metrics::NoOpMetricsCollector};
-    use lance_index::{optimize::OptimizeOptions, scalar::IndexReader};
     use lance_io::{
         object_store::ObjectStore,
         scheduler::{ScanScheduler, SchedulerConfig},
@@ -2757,6 +3014,17 @@ mod tests {
             .collect()
     }
 
+    async fn load_flat_partition_row_ids(index: &IvfFlatIndex, partition_idx: usize) -> Vec<u64> {
+        index
+            .storage
+            .load_partition(partition_idx, None)
+            .await
+            .unwrap()
+            .row_ids()
+            .copied()
+            .collect()
+    }
+
     async fn delete_ids(dataset: &mut Dataset, ids: &[u64]) {
         if ids.is_empty() {
             return;
@@ -4411,17 +4679,19 @@ mod tests {
         test_index_impl::<Float32Type>(params, nlist, 0.75, -1.0..1.0, None).await;
     }
 
-    // RQ doesn't perform well for random data
-    // need to verify recall with real-world dataset (e.g. sift1m)
+    // These queries probe every partition, so recall here measures RaBitQ quantization
+    // error alone. At 1 bit per dimension it averages ~0.67 on this uniformly random,
+    // L2-normalized data, and each build draws a fresh random rotation, so no bar worth
+    // asserting sits clear of the spread. 5 bits lifts recall to ~0.97; its `ex_bits = 4`
+    // also covers a FastScan ex-code kernel that the multi-bit test below never reaches.
     #[rstest]
-    #[case(1, DistanceType::L2, 0.5)]
-    #[case(1, DistanceType::Cosine, 0.5)]
-    #[case(1, DistanceType::Dot, 0.5)]
-    #[case(4, DistanceType::L2, 0.5)]
-    #[case(4, DistanceType::Cosine, 0.5)]
-    #[case(4, DistanceType::Dot, 0.5)]
+    #[case(1, DistanceType::L2, 0.9)]
+    #[case(1, DistanceType::Cosine, 0.9)]
+    #[case(1, DistanceType::Dot, 0.9)]
+    #[case(4, DistanceType::L2, 0.9)]
+    #[case(4, DistanceType::Cosine, 0.9)]
+    #[case(4, DistanceType::Dot, 0.9)]
     #[tokio::test]
-    // #[ignore = "Temporarily skipping flaky 4-bit IVF_RQ tests"]
     async fn test_build_ivf_rq(
         #[case] nlist: usize,
         #[case] distance_type: DistanceType,
@@ -4430,7 +4700,7 @@ mod tests {
     ) {
         let _ = env_logger::try_init();
         let ivf_params = IvfBuildParams::new(nlist);
-        let rq_params = RQBuildParams::with_rotation_type(1, rotation_type);
+        let rq_params = RQBuildParams::with_rotation_type(5, rotation_type);
         let params = VectorIndexParams::with_ivf_rq_params(distance_type, ivf_params, rq_params);
         test_index(params.clone(), nlist, recall_requirement, None).await;
         if distance_type == DistanceType::Cosine {
@@ -4717,10 +4987,7 @@ mod tests {
             .as_primitive::<Float32Type>()
             .values()
             .to_vec();
-        let results = dists
-            .into_iter()
-            .zip(row_ids.clone().into_iter())
-            .collect::<Vec<_>>();
+        let results = dists.into_iter().zip(row_ids.clone()).collect::<Vec<_>>();
         let row_ids = row_ids.into_iter().collect::<HashSet<_>>();
 
         let gt = multivec_ground_truth(&vectors, &query, k, params.metric_type);
@@ -5113,10 +5380,7 @@ mod tests {
             .as_primitive::<Float32Type>()
             .values()
             .to_vec();
-        let results = dists
-            .into_iter()
-            .zip(row_ids.into_iter())
-            .collect::<Vec<_>>();
+        let results = dists.into_iter().zip(row_ids).collect::<Vec<_>>();
         let row_ids = results.iter().map(|(_, id)| *id).collect::<HashSet<_>>();
         assert!(row_ids.len() == k);
 
@@ -5172,9 +5436,22 @@ mod tests {
 
         // Rewrite auxiliary file with PQ codebook inlined into schema metadata.
         let mut metadata = reader.schema().metadata.clone();
-        let batch = reader
-            .read_range(0..reader.num_rows() as usize, None)
+        let projection = lance_file::reader::ReaderProjection::from_whole_schema(
+            reader.schema(),
+            reader.metadata().version(),
+        );
+        let batches = reader
+            .read_stream_projected(
+                lance_io::ReadBatchParams::RangeFull,
+                u32::MAX,
+                u32::MAX,
+                projection,
+                lance_encoding::decoder::FilterExpression::no_filter(),
+            )
             .await?;
+        use futures::TryStreamExt as _;
+        let batches = batches.try_collect::<Vec<_>>().await?;
+        let batch = arrow::compute::concat_batches(&batches[0].schema(), &batches)?;
         let new_aux_path = new_dir.clone().join(INDEX_AUXILIARY_FILE_NAME);
         let mut writer = FileWriter::try_new(
             obj_store.create(&new_aux_path).await?,
@@ -6024,6 +6301,187 @@ mod tests {
         }
     }
 
+    async fn row_ids_matching(dataset: &Dataset, predicate: &str) -> HashSet<u64> {
+        let mut scan = dataset.scan();
+        scan.with_row_id();
+        scan.filter(predicate).unwrap();
+        let batch = scan.try_into_batch().await.unwrap();
+        batch[ROW_ID]
+            .as_primitive::<UInt64Type>()
+            .values()
+            .iter()
+            .copied()
+            .collect()
+    }
+
+    struct OptimizeAfterDelete {
+        deleted_row_ids: HashSet<u64>,
+        partition_row_ids_before: Vec<Vec<u64>>,
+        index_row_ids: HashSet<u64>,
+        num_partitions_after: usize,
+        stats_json: String,
+    }
+
+    /// Shared scenario for the issue-7701 regressions: stable-row-id dataset,
+    /// IVF_FLAT index, scattered delete, optimize. Asserts the invariants both
+    /// partition adjustments must hold -- no live row lost, no id that never
+    /// existed -- and returns the state for the mode-specific assertions.
+    async fn optimize_after_delete(
+        total_rows: usize,
+        nlist: usize,
+        delete_predicate: &str,
+        keep_predicate: &str,
+    ) -> OptimizeAfterDelete {
+        const INDEX_NAME: &str = "vector_idx";
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let (batch, schema) = generate_batch::<Float32Type>(total_rows, None, 0.0..1.0, false);
+        let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+        let mut dataset = Dataset::write(
+            batches,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = VectorIndexParams::ivf_flat(nlist, DistanceType::L2);
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some(INDEX_NAME.to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let index_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await;
+        let flat = index_ctx
+            .index
+            .as_any()
+            .downcast_ref::<IvfFlatIndex>()
+            .expect("expected IvfFlat index");
+        let mut partition_row_ids_before = Vec::with_capacity(nlist);
+        for part in 0..flat.ivf.num_partitions() {
+            partition_row_ids_before.push(load_flat_partition_row_ids(flat, part).await);
+        }
+
+        let deleted_row_ids = row_ids_matching(&dataset, delete_predicate).await;
+        let live_row_ids = row_ids_matching(&dataset, keep_predicate).await;
+        dataset.delete(delete_predicate).await.unwrap();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::new())
+            .await
+            .unwrap();
+
+        let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await;
+        let num_partitions_after = final_ctx.num_partitions();
+        let stats_json = final_ctx.stats_json().to_string();
+        let flat = final_ctx
+            .index
+            .as_any()
+            .downcast_ref::<IvfFlatIndex>()
+            .expect("expected IvfFlat index");
+        let mut index_row_ids = HashSet::new();
+        for part in 0..flat.ivf.num_partitions() {
+            index_row_ids.extend(load_flat_partition_row_ids(flat, part).await);
+        }
+
+        for row_id in &live_row_ids {
+            assert!(
+                index_row_ids.contains(row_id),
+                "live row id {} missing from index after optimize",
+                row_id
+            );
+        }
+        for row_id in &index_row_ids {
+            assert!(
+                live_row_ids.contains(row_id) || deleted_row_ids.contains(row_id),
+                "unexpected row id {} in index after optimize",
+                row_id
+            );
+        }
+
+        OptimizeAfterDelete {
+            deleted_row_ids,
+            partition_row_ids_before,
+            index_row_ids,
+            num_partitions_after,
+            stats_json,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_optimize_join_after_delete_with_stable_row_ids() {
+        // Regression test for https://github.com/lance-format/lance/issues/7701:
+        // every partition (400 rows / 4) is under the IVF_FLAT join threshold,
+        // so optimize joins the smallest after a scattered delete.
+        let run = optimize_after_delete(400, 4, "id % 3 = 0", "id % 3 != 0").await;
+
+        assert_eq!(
+            run.num_partitions_after, 3,
+            "optimize should have joined the smallest partition, got stats: {}",
+            run.stats_json
+        );
+
+        // Only the joined partition is rebuilt from live rows; untouched
+        // partitions keep their deleted ids (filtered at query time).
+        let missing = run
+            .deleted_row_ids
+            .difference(&run.index_row_ids)
+            .copied()
+            .collect::<HashSet<_>>();
+        assert!(
+            !missing.is_empty(),
+            "joined partition should have contained deleted row ids"
+        );
+        let matches_one_partition = run.partition_row_ids_before.iter().any(|part_ids| {
+            let part_deleted = part_ids
+                .iter()
+                .copied()
+                .filter(|id| run.deleted_row_ids.contains(id))
+                .collect::<HashSet<_>>();
+            part_deleted == missing
+        });
+        assert!(
+            matches_one_partition,
+            "row ids dropped from the index should be exactly the joined partition's deleted ids"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_optimize_split_after_delete_with_stable_row_ids() {
+        // Regression test for https://github.com/lance-format/lance/issues/7701:
+        // one partition holds more than 4x the IVF_FLAT target, so optimize
+        // splits it after a scattered delete. This path reaches
+        // filter_deleted_ids through reshuffle_partitions, unlike the join
+        // path's take_vectors.
+        let run = optimize_after_delete(20_000, 1, "id % 5 = 0", "id % 5 != 0").await;
+
+        assert!(
+            run.num_partitions_after > 1,
+            "optimize should have split the oversized partition, got stats: {}",
+            run.stats_json
+        );
+
+        // The split rebuilds the whole partition from live rows: no deleted
+        // ids remain.
+        for row_id in &run.deleted_row_ids {
+            assert!(
+                !run.index_row_ids.contains(row_id),
+                "deleted row id {} still in index after split",
+                row_id
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_prewarm_ivf_pq() {
         use lance_io::assert_io_eq;
diff --git a/rust/lance/src/index/vector/utils.rs b/rust/lance/src/index/vector/utils.rs
index 3046d0f3a83..012d8d3a703 100644
--- a/rust/lance/src/index/vector/utils.rs
+++ b/rust/lance/src/index/vector/utils.rs
@@ -1,16 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashSet;
 use std::pin::Pin;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use arrow::array::ArrayData;
 use arrow::datatypes::DataType;
 use arrow_array::new_empty_array;
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, cast::AsArray};
 use arrow_buffer::{Buffer, MutableBuffer};
-use futures::{Stream, StreamExt, TryStreamExt, stream};
+use futures::{Stream, StreamExt, stream};
 use lance_arrow::DataTypeExt;
 use lance_core::datatypes::Schema;
 use lance_linalg::distance::DistanceType;
@@ -18,6 +18,7 @@ use log::{info, warn};
 use rand::rngs::SmallRng;
 use rand::seq::{IteratorRandom, SliceRandom};
 use rand::{Rng, SeedableRng};
+use roaring::RoaringTreemap;
 use tokio::sync::Mutex;
 
 use crate::dataset::{Dataset, ProjectionRequest, TakeBuilder, row_offsets_to_row_addresses};
@@ -264,38 +265,7 @@ fn infer_vector_element_type_impl(
 async fn count_rows(dataset: &Dataset, fragment_ids: Option<&[u32]>) -> Result<usize> {
     match fragment_ids {
         None => dataset.count_rows(None).await,
-        Some(fragment_ids) => {
-            let sorted_ids: Vec<u32>;
-            let sorted_fragment_ids = if fragment_ids.windows(2).all(|w| w[0] <= w[1]) {
-                fragment_ids
-            } else {
-                sorted_ids = {
-                    let mut v = fragment_ids.to_vec();
-                    v.sort_unstable();
-                    v
-                };
-                &sorted_ids
-            };
-            let fragments = dataset.get_frags_from_ordered_ids(sorted_fragment_ids);
-            let valid_fragments = fragments
-                .into_iter()
-                .enumerate()
-                .map(|(i, frag)| {
-                    frag.ok_or_else(|| {
-                        Error::index(format!(
-                            "Unexpectedly missing fragment {}",
-                            sorted_fragment_ids[i]
-                        ))
-                    })
-                })
-                .collect::<Result<Vec<_>>>()?;
-            let cnts = stream::iter(valid_fragments)
-                .map(|f| async move { f.count_rows(None).await })
-                .buffer_unordered(16)
-                .try_collect::<Vec<usize>>()
-                .await?;
-            Ok(cnts.iter().sum::<usize>())
-        }
+        Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await,
     }
 }
 
@@ -309,8 +279,6 @@ pub async fn maybe_sample_training_data(
     sample_size_hint: usize,
     fragment_ids: Option<&[u32]>,
 ) -> Result<FixedSizeListArray> {
-    let num_rows = count_rows(dataset, fragment_ids).await?;
-
     let vector_field = dataset.schema().field(column).ok_or(Error::index(format!(
         "Sample training data: column {} does not exist in schema",
         column
@@ -328,6 +296,8 @@ pub async fn maybe_sample_training_data(
         return Ok(new_empty_array(&fsl_type).as_fixed_size_list().clone());
     }
 
+    let num_rows = count_rows(dataset, fragment_ids).await?;
+
     let is_nullable = vector_field.nullable;
 
     let sample_size_hint = match vector_field.data_type() {
@@ -486,18 +456,39 @@ async fn sample_training_data(
             );
             return vector_column_to_fsl(&batch, column);
         }
+        // Rows the consumer still needs. The fragment producer sizes each
+        // prefetch round to this outstanding demand, keeping reads bounded by
+        // the requested sample size.
+        let still_needed = Arc::new(AtomicUsize::new(sample_size_hint));
         let scan = sample_training_data_scan_from_fragments(
             dataset,
             column,
-            sample_size_hint,
             num_rows,
             fragment_ids,
+            still_needed.clone(),
         )?;
         return match vector_field.data_type() {
             DataType::FixedSizeList(_, _) => {
-                sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await
+                sample_nullable_fsl(
+                    column,
+                    sample_size_hint,
+                    byte_width,
+                    vector_field,
+                    scan,
+                    Some(still_needed),
+                )
+                .await
+            }
+            _ => {
+                sample_nullable_fallback(
+                    column,
+                    sample_size_hint,
+                    is_nullable,
+                    scan,
+                    Some(still_needed),
+                )
+                .await
             }
-            _ => sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await,
         };
     }
 
@@ -516,12 +507,20 @@ async fn sample_training_data(
         DataType::FixedSizeList(_, _) => {
             let scan =
                 sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?;
-            sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await
+            sample_nullable_fsl(
+                column,
+                sample_size_hint,
+                byte_width,
+                vector_field,
+                scan,
+                None,
+            )
+            .await
         }
         _ => {
             let scan =
                 sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?;
-            sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await
+            sample_nullable_fallback(column, sample_size_hint, is_nullable, scan, None).await
         }
     }
 }
@@ -550,12 +549,19 @@ fn sample_training_data_scan(
 /// sampling must first map random offsets within the selected fragments to row
 /// addresses and then `take` those rows. Both nullable FSL and multivector
 /// paths reuse this stream to avoid duplicating fragment sampling logic.
+///
+/// Each round is sized to `still_needed` (the consumer's outstanding demand),
+/// so a low-null column reads at most the requested sample. Visited offsets
+/// are tracked in a [`RoaringTreemap`] because sparse or all-null columns
+/// force the stream to visit most selected rows before it can terminate, and
+/// a compressed bitmap keeps that persistent state near `num_rows / 8` bytes
+/// even when fully populated.
 fn sample_training_data_scan_from_fragments(
     dataset: &Dataset,
     column: &str,
-    sample_size_hint: usize,
     num_rows: usize,
     fragment_ids: &[u32],
+    still_needed: Arc<AtomicUsize>,
 ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
     if fragment_ids.is_empty() {
         return Err(Error::invalid_input(
@@ -563,21 +569,7 @@ fn sample_training_data_scan_from_fragments(
         ));
     }
 
-    let mut ordered_ids = fragment_ids.to_vec();
-    ordered_ids.sort_unstable();
-    ordered_ids.dedup();
-    let selected_fragments = dataset
-        .get_frags_from_ordered_ids(&ordered_ids)
-        .into_iter()
-        .zip(ordered_ids.iter())
-        .map(|(fragment, fragment_id)| {
-            fragment.ok_or_else(|| {
-                Error::invalid_input(format!(
-                    "Unknown fragment id {fragment_id} in training fragment filter"
-                ))
-            })
-        })
-        .collect::<Result<Vec<_>>>()?;
+    let selected_fragments = dataset.get_fragments_from_ids(fragment_ids)?;
     let dataset = Arc::new(dataset.clone());
     let projection = Arc::new(
         ProjectionRequest::from(dataset.schema().project(&[column])?)
@@ -589,19 +581,36 @@ fn sample_training_data_scan_from_fragments(
             dataset,
             projection,
             selected_fragments,
-            HashSet::<u64>::with_capacity(sample_size_hint.min(num_rows)),
+            RoaringTreemap::new(),
             SmallRng::from_os_rng(),
+            still_needed,
         ),
-        move |(dataset, projection, selected_fragments, mut seen_offsets, mut rng)| async move {
-            if seen_offsets.len() >= num_rows {
+        move |(
+            dataset,
+            projection,
+            selected_fragments,
+            mut seen_offsets,
+            mut rng,
+            still_needed,
+        )| async move {
+            if seen_offsets.len() as usize >= num_rows {
+                return Ok(None);
+            }
+            let still = still_needed.load(Ordering::Relaxed);
+            if still == 0 {
                 return Ok(None);
             }
 
-            let remaining = num_rows.saturating_sub(seen_offsets.len());
-            let target = sample_size_hint.saturating_mul(2).min(remaining);
+            let remaining = num_rows.saturating_sub(seen_offsets.len() as usize);
+            // Sizing the round to the outstanding demand keeps a low-null
+            // column's reads bounded by the requested sample, matching the
+            // non-nullable path.
+            let target = still.min(remaining);
             let mut sampled_offsets = if remaining <= target.saturating_mul(4) {
+                // Few offsets remain unseen, so shuffling the unseen set is
+                // cheaper than repeatedly rejecting already-sampled offsets.
                 let mut unseen_indices = (0..num_rows as u64)
-                    .filter(|index| !seen_offsets.contains(index))
+                    .filter(|index| !seen_offsets.contains(*index))
                     .collect::<Vec<_>>();
                 unseen_indices.shuffle(&mut rng);
                 unseen_indices.truncate(target);
@@ -635,7 +644,14 @@ fn sample_training_data_scan_from_fragments(
             .await?;
             Ok(Some((
                 batch,
-                (dataset, projection, selected_fragments, seen_offsets, rng),
+                (
+                    dataset,
+                    projection,
+                    selected_fragments,
+                    seen_offsets,
+                    rng,
+                    still_needed,
+                ),
             )))
         },
     );
@@ -646,22 +662,7 @@ fn resolve_scan_fragments(
     dataset: &Dataset,
     fragment_ids: &[u32],
 ) -> Result<Vec<lance_table::format::Fragment>> {
-    let mut ordered_ids = fragment_ids.to_vec();
-    ordered_ids.sort_unstable();
-    let fragments = dataset.get_frags_from_ordered_ids(&ordered_ids);
-    if let Some(missing_id) = fragments
-        .iter()
-        .zip(ordered_ids.iter())
-        .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id))
-    {
-        return Err(Error::invalid_input(format!(
-            "Unknown fragment id {missing_id} in training fragment filter"
-        )));
-    }
-    Ok(fragments
-        .into_iter()
-        .map(|fragment| fragment.unwrap().metadata().clone())
-        .collect())
+    dataset.get_fragment_metadata_from_ids(fragment_ids)
 }
 
 /// Build a FixedSizeListArray from raw flat value bytes.
@@ -721,6 +722,7 @@ async fn sample_nullable_fsl<S>(
     byte_width: usize,
     vector_field: &lance_core::datatypes::Field,
     mut scan: S,
+    still_needed: Option<Arc<AtomicUsize>>,
 ) -> Result<FixedSizeListArray>
 where
     S: Stream<Item = Result<RecordBatch>> + Unpin,
@@ -731,6 +733,12 @@ where
     let mut rows_scanned: usize = 0;
 
     while num_non_null < sample_size_hint {
+        let remaining_rows = sample_size_hint - num_non_null;
+        // A fragment-limited producer sizes its next prefetch round to this
+        // outstanding demand.
+        if let Some(still_needed) = &still_needed {
+            still_needed.store(remaining_rows, Ordering::Relaxed);
+        }
         let Some(batch) = scan.next().await else {
             break;
         };
@@ -750,7 +758,16 @@ where
             continue;
         }
         let previous_num_non_null = num_non_null;
-        accumulate_fsl_values(&mut values_buf, &mut num_non_null, &array, byte_width, true)?;
+        // `remaining_rows` keeps `values_buf` within its pre-allocated
+        // `sample_size_hint * byte_width` capacity.
+        accumulate_fsl_values(
+            &mut values_buf,
+            &mut num_non_null,
+            &array,
+            byte_width,
+            true,
+            remaining_rows,
+        )?;
         info!(
             "Sample training data: batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled after null filtering)",
             batch_count,
@@ -762,6 +779,11 @@ where
         );
     }
 
+    // Zero the demand so any further poll of the producer terminates instead
+    // of reading another round.
+    if let Some(still_needed) = &still_needed {
+        still_needed.store(0, Ordering::Relaxed);
+    }
     let num_rows_out = num_non_null.min(sample_size_hint);
     values_buf.truncate(num_rows_out * byte_width);
 
@@ -797,7 +819,14 @@ async fn sample_fsl_uniform(
     for (chunk_idx, chunk) in indices.chunks(TAKE_CHUNK_SIZE).enumerate() {
         let batch = dataset.take(chunk, projection.clone()).await?;
         let array = get_column_from_batch(&batch, column)?;
-        accumulate_fsl_values(&mut values_buf, &mut total_rows, &array, byte_width, false)?;
+        accumulate_fsl_values(
+            &mut values_buf,
+            &mut total_rows,
+            &array,
+            byte_width,
+            false,
+            usize::MAX,
+        )?;
         info!(
             "Sample training data: batch {}/{} read {} rows ({}/{} sampled by uniform random sampling)",
             chunk_idx + 1,
@@ -821,13 +850,20 @@ async fn sample_fsl_uniform(
 /// When `filter_nulls` is false and there are no nulls, copies raw bytes
 /// directly from the FSL values buffer (accounting for child array offset).
 /// When `filter_nulls` is true, uses Arrow's `filter` kernel to remove nulls.
+/// At most `max_rows` rows are appended so callers can stop copying once their
+/// sample is full; otherwise one oversized source batch can grow `values_buf`
+/// far beyond the intended cap.
 fn accumulate_fsl_values(
     values_buf: &mut MutableBuffer,
     num_rows: &mut usize,
     array: &ArrayRef,
     byte_width: usize,
     filter_nulls: bool,
+    max_rows: usize,
 ) -> Result<()> {
+    if max_rows == 0 {
+        return Ok(());
+    }
     let needs_filter = filter_nulls && array.null_count() > 0;
 
     if needs_filter {
@@ -835,21 +871,29 @@ fn accumulate_fsl_values(
         let mask = arrow_array::BooleanArray::from(nulls.inner().clone());
         let filtered = arrow::compute::filter(array, &mask)?;
         let fsl = filtered.as_fixed_size_list();
+        let take = fsl.len().min(max_rows);
+        if take == 0 {
+            return Ok(());
+        }
         let values_data = fsl.values().to_data();
-        let value_bytes = &values_data.buffers()[0].as_slice()[..fsl.len() * byte_width];
+        let value_bytes = &values_data.buffers()[0].as_slice()[..take * byte_width];
         values_buf.extend_from_slice(value_bytes);
-        *num_rows += fsl.len();
+        *num_rows += take;
     } else {
         // No nulls: copy raw bytes directly, accounting for child array offset.
         let fsl = array.as_fixed_size_list();
+        let take = fsl.len().min(max_rows);
+        if take == 0 {
+            return Ok(());
+        }
         let values = fsl.values();
         let values_data = values.to_data();
         let elem_size = byte_width / fsl.value_length() as usize;
         let offset_bytes = values_data.offset() * elem_size;
-        let total_bytes = fsl.len() * byte_width;
+        let total_bytes = take * byte_width;
         let buf = &values_data.buffers()[0].as_slice()[offset_bytes..offset_bytes + total_bytes];
         values_buf.extend_from_slice(buf);
-        *num_rows += fsl.len();
+        *num_rows += take;
     }
     Ok(())
 }
@@ -862,6 +906,7 @@ async fn sample_nullable_fallback<S>(
     sample_size_hint: usize,
     is_nullable: bool,
     mut scan: S,
+    still_needed: Option<Arc<AtomicUsize>>,
 ) -> Result<FixedSizeListArray>
 where
     S: Stream<Item = Result<RecordBatch>> + Unpin,
@@ -873,6 +918,12 @@ where
     let mut rows_scanned: usize = 0;
 
     while num_non_null < sample_size_hint {
+        let remaining_rows = sample_size_hint - num_non_null;
+        // A fragment-limited producer sizes its next prefetch round to this
+        // outstanding demand.
+        if let Some(still_needed) = &still_needed {
+            still_needed.store(remaining_rows, Ordering::Relaxed);
+        }
         let Some(batch) = scan.next().await else {
             break;
         };
@@ -898,7 +949,14 @@ where
         } else {
             batch
         };
-        let accepted_rows = batch.num_rows();
+        // Slicing to the outstanding demand keeps the retained batches, and
+        // the post-loop `concat_batches`, bounded by the sample size.
+        let accepted_rows = batch.num_rows().min(remaining_rows);
+        let batch = if accepted_rows < batch.num_rows() {
+            batch.slice(0, accepted_rows)
+        } else {
+            batch
+        };
         num_non_null += accepted_rows;
         info!(
             "Sample training data (fallback): batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled)",
@@ -912,6 +970,12 @@ where
         filtered.push(batch);
     }
 
+    // Zero the demand so any further poll of the producer terminates instead
+    // of reading another round.
+    if let Some(still_needed) = &still_needed {
+        still_needed.store(0, Ordering::Relaxed);
+    }
+
     let Some(schema) = schema else {
         return Err(Error::index("No non-null training data found".to_string()));
     };
@@ -1040,6 +1104,7 @@ mod tests {
 
     use crate::dataset::InsertBuilder;
     use arrow_array::{ArrayRef, Float32Array, types::Float32Type};
+    use arrow_buffer::{BooleanBufferBuilder, NullBuffer};
     use arrow_schema::{DataType, Field};
     use lance_arrow::FixedSizeListArrayExt;
     use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch};
@@ -1201,7 +1266,15 @@ mod tests {
         let mut buf = MutableBuffer::new(0);
         let mut num_rows = 0usize;
         let sliced_ref: ArrayRef = Arc::new(sliced);
-        accumulate_fsl_values(&mut buf, &mut num_rows, &sliced_ref, byte_width, false).unwrap();
+        accumulate_fsl_values(
+            &mut buf,
+            &mut num_rows,
+            &sliced_ref,
+            byte_width,
+            false,
+            usize::MAX,
+        )
+        .unwrap();
 
         assert_eq!(num_rows, 4);
         let result: &[f32] =
@@ -1327,4 +1400,140 @@ mod tests {
         let result = count_rows(&dataset, Some(&[ids[2], ids[0]])).await.unwrap();
         assert_eq!(result, 250);
     }
+
+    /// Nullable FSL with fragment-limited sampling must fill the requested sample
+    /// size when enough non-null rows exist, and terminate cleanly when all
+    /// selected rows are null.
+    #[tokio::test]
+    async fn test_maybe_sample_training_data_fsl_nullable_fragment_limited() {
+        let nrows: usize = 2000;
+        let dims: u32 = 8;
+        let sample_size: usize = 500;
+
+        for (case, null_probability, expected_len) in
+            [("partial_nulls", 0.5, sample_size), ("all_nulls", 1.0, 0)]
+        {
+            let col_gen = array::rand_vec::<Float32Type>(Dimension::from(dims))
+                .with_random_nulls(null_probability);
+            let data = gen_batch()
+                .col("vec", col_gen)
+                .into_batch_rows(RowCount::from(nrows as u64))
+                .unwrap();
+
+            let dataset = InsertBuilder::new("memory://")
+                .execute(vec![data])
+                .await
+                .unwrap();
+
+            let fragment_ids: Vec<u32> = dataset
+                .get_fragments()
+                .iter()
+                .map(|f| f.id() as u32)
+                .collect();
+
+            let training_data =
+                maybe_sample_training_data(&dataset, "vec", sample_size, Some(&fragment_ids))
+                    .await
+                    .unwrap();
+
+            assert_eq!(training_data.len(), expected_len, "{case}");
+            assert_eq!(training_data.null_count(), 0, "{case}");
+            assert_eq!(training_data.value_length(), dims as i32, "{case}");
+        }
+    }
+
+    /// Scan-side regression: each fragment-limited producer round must read at
+    /// most the consumer's outstanding demand. Driving the producer directly
+    /// with a fixed `still_needed` and inspecting the raw batch size catches
+    /// over-reads that a post-truncation output-length check cannot.
+    #[tokio::test]
+    async fn test_sample_fragment_scan_round_caps_at_still_needed() {
+        let nrows: usize = 4000;
+        let dims: u32 = 8;
+        let still: usize = 500;
+
+        let col_gen = array::rand_vec::<Float32Type>(Dimension::from(dims)).with_random_nulls(0.5);
+        let data = gen_batch()
+            .col("vec", col_gen)
+            .into_batch_rows(RowCount::from(nrows as u64))
+            .unwrap();
+
+        let dataset = InsertBuilder::new("memory://fsl_scan_round_cap_test")
+            .execute(vec![data])
+            .await
+            .unwrap();
+
+        let fragment_ids: Vec<u32> = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect();
+        let num_rows = count_rows(&dataset, Some(&fragment_ids)).await.unwrap();
+
+        // `still_needed` is left large enough that `num_rows` never bounds the
+        // round, so the batch size reflects the demand cap and nothing else.
+        let still_needed = Arc::new(AtomicUsize::new(still));
+        let mut scan = sample_training_data_scan_from_fragments(
+            &dataset,
+            "vec",
+            num_rows,
+            &fragment_ids,
+            still_needed.clone(),
+        )
+        .unwrap();
+
+        let batch = scan.next().await.unwrap().unwrap();
+        assert!(
+            batch.num_rows() <= still,
+            "producer round read {} rows but only {} were outstanding",
+            batch.num_rows(),
+            still
+        );
+    }
+
+    #[test]
+    fn test_accumulate_fsl_values_respects_max_rows() {
+        let dim: usize = 4;
+        let total_rows: usize = 100;
+        let max_rows: usize = 16;
+        let byte_width = dim * std::mem::size_of::<f32>();
+
+        let values: Vec<f32> = (0..total_rows * dim).map(|i| i as f32).collect();
+        let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim as i32)
+            .unwrap();
+        let arr: ArrayRef = Arc::new(fsl);
+
+        let mut buf = MutableBuffer::new(0);
+        let mut num_rows = 0usize;
+        accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap();
+
+        assert_eq!(num_rows, max_rows);
+        assert_eq!(buf.len(), max_rows * byte_width);
+
+        let values: Vec<f32> = (0..total_rows * dim).map(|i| i as f32).collect();
+        let item_field = Arc::new(Field::new("item", DataType::Float32, true));
+
+        // Every other row is null, leaving 50 non-null rows.
+        let mut nulls_builder = BooleanBufferBuilder::new(total_rows);
+        for i in 0..total_rows {
+            nulls_builder.append(i % 2 == 0);
+        }
+        let nulls = NullBuffer::new(nulls_builder.finish());
+
+        let fsl = FixedSizeListArray::try_new(
+            item_field,
+            dim as i32,
+            Arc::new(Float32Array::from(values)),
+            Some(nulls),
+        )
+        .unwrap();
+        let arr: ArrayRef = Arc::new(fsl);
+
+        let mut buf = MutableBuffer::new(0);
+        let mut num_rows = 0usize;
+        accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap();
+
+        assert_eq!(num_rows, max_rows);
+        assert_eq!(buf.len(), max_rows * byte_width);
+    }
 }
diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs
index 6ac6a0c362f..2441e31749d 100644
--- a/rust/lance/src/io/commit.rs
+++ b/rust/lance/src/io/commit.rs
@@ -140,6 +140,7 @@ pub(crate) async fn write_transaction_file(
 #[allow(clippy::too_many_arguments)]
 async fn do_commit_new_dataset(
     object_store: &ObjectStore,
+    source_store: Option<&ObjectStore>,
     commit_handler: &dyn CommitHandler,
     base_path: &Path,
     transaction: &Transaction,
@@ -163,15 +164,19 @@ async fn do_commit_new_dataset(
         ..
     } = &transaction.operation
     {
+        // The source manifest must be read through the source store, which may differ
+        // from the destination store when cloning across object stores/accounts. Falls
+        // back to the destination store for same-store clones.
+        let source_store = source_store.unwrap_or(object_store);
         let source_base_path =
             ObjectStore::extract_path_from_uri(store_registry, ref_path.as_str())?;
         let source_manifest_location = commit_handler
-            .resolve_version_location(&source_base_path, *ref_version, &object_store.inner)
+            .resolve_version_location(&source_base_path, *ref_version, &source_store.inner)
             .await?;
         let source_manifest = Dataset::load_manifest(
-            object_store,
+            source_store,
             &source_manifest_location,
-            base_path.to_string().as_str(),
+            ref_path.as_str(),
             &Session::default(),
         )
         .await?;
@@ -192,7 +197,7 @@ async fn do_commit_new_dataset(
             );
 
             let updated_indices = if let Some(index_section_pos) = source_manifest.index_section {
-                let reader = object_store.open(&source_manifest_location.path).await?;
+                let reader = source_store.open(&source_manifest_location.path).await?;
                 let section: pb::IndexSection =
                     lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?;
                 section
@@ -229,7 +234,7 @@ async fn do_commit_new_dataset(
             // Indices: keep metadata but normalize base to local
             let mut updated_indices = Vec::new();
             if let Some(index_section_pos) = source_manifest.index_section {
-                let reader = object_store.open(&source_manifest_location.path).await?;
+                let reader = source_store.open(&source_manifest_location.path).await?;
                 let section: pb::IndexSection =
                     lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?;
                 updated_indices = section
@@ -300,6 +305,7 @@ async fn do_commit_new_dataset(
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn commit_new_dataset(
     object_store: &ObjectStore,
+    source_store: Option<&ObjectStore>,
     commit_handler: &dyn CommitHandler,
     base_path: &Path,
     transaction: &Transaction,
@@ -310,6 +316,7 @@ pub(crate) async fn commit_new_dataset(
 ) -> Result<(Manifest, ManifestLocation)> {
     do_commit_new_dataset(
         object_store,
+        source_store,
         commit_handler,
         base_path,
         transaction,
@@ -489,13 +496,12 @@ fn fix_schema(manifest: &mut Manifest) -> Result<()> {
     }
 
     // Now, we need to remap the field ids to be unique.
-    let mut field_id_seed = manifest.max_field_id() + 1;
     let mut old_field_id_mapping: HashMap<i32, i32> = HashMap::new();
     let mut fields_with_duplicate_ids = fields_with_duplicate_ids.into_iter().collect::<Vec<_>>();
     fields_with_duplicate_ids.sort_unstable();
-    for field_id in fields_with_duplicate_ids {
+    for (field_id_seed, field_id) in (manifest.max_field_id() + 1..).zip(fields_with_duplicate_ids)
+    {
         old_field_id_mapping.insert(field_id, field_id_seed);
-        field_id_seed += 1;
     }
 
     let mut fragments = manifest.fragments.as_ref().clone();
@@ -1074,6 +1080,7 @@ pub(crate) async fn commit_transaction(
                 if !indices.is_empty() {
                     let key = IndexMetadataKey {
                         version: target_version,
+                        store_identity: &dataset.object_store.store_prefix,
                     };
                     dataset
                         .index_cache
@@ -1691,6 +1698,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path1", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("unused", vec![9], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1703,6 +1711,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("path3", vec![2], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1740,6 +1749,7 @@ mod tests {
                     vec![0, 1, 10],
                     None,
                 )],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1752,6 +1762,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("path3", vec![10], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1842,6 +1853,7 @@ mod tests {
         let fragment = Fragment {
             id: 0,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(100),
diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs
index d95821dd130..01341b41552 100644
--- a/rust/lance/src/io/commit/conflict_resolver.rs
+++ b/rust/lance/src/io/commit/conflict_resolver.rs
@@ -7,15 +7,17 @@ use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta};
 use crate::io::deletion::read_dataset_deletion_file;
 use crate::{
     Dataset,
-    dataset::transaction::{Operation, Transaction, UpdateMode},
+    dataset::transaction::{DataOverlayGroup, Operation, Transaction, UpdateMode},
 };
 use futures::{StreamExt, TryStreamExt};
 use lance_core::{Error, Result, utils::deletion::DeletionVector};
 use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
-use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MergedGeneration};
+use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME};
 use lance_select::{RowAddrTreeMap, RowSetOps};
 use lance_table::format::IndexMetadata;
+use lance_table::format::overlay::OverlayCoverage;
 use lance_table::{format::Fragment, io::deletion::write_deletion_file};
+use roaring::RoaringBitmap;
 use std::{
     borrow::Cow,
     collections::{HashMap, HashSet},
@@ -32,9 +34,9 @@ pub struct TransactionRebase<'a> {
     modified_fragment_ids: HashSet<u64>,
     affected_rows: Option<&'a RowAddrTreeMap>,
     conflicting_frag_reuse_indices: Vec<IndexMetadata>,
-    /// Merged generations from conflicting UpdateMemWalState transactions.
+    /// Compacted SSTables from conflicting UpdateMemWalState transactions.
     /// Used when rebasing CreateIndex of MemWalIndex.
-    conflicting_mem_wal_merged_gens: Vec<MergedGeneration>,
+    conflicting_mem_wal_compacted_sstables: Vec<CompactedSsTable>,
 }
 
 impl<'a> TransactionRebase<'a> {
@@ -60,7 +62,7 @@ impl<'a> TransactionRebase<'a> {
                 initial_fragments: HashMap::new(),
                 modified_fragment_ids: HashSet::new(),
                 conflicting_frag_reuse_indices: Vec::new(),
-                conflicting_mem_wal_merged_gens: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
             }),
             Operation::Delete {
                 updated_fragments,
@@ -88,7 +90,7 @@ impl<'a> TransactionRebase<'a> {
                         modified_fragment_ids,
                         affected_rows: None,
                         conflicting_frag_reuse_indices: Vec::new(),
-                        conflicting_mem_wal_merged_gens: Vec::new(),
+                        conflicting_mem_wal_compacted_sstables: Vec::new(),
                     });
                 }
 
@@ -101,7 +103,7 @@ impl<'a> TransactionRebase<'a> {
                     initial_fragments,
                     modified_fragment_ids,
                     conflicting_frag_reuse_indices: Vec::new(),
-                    conflicting_mem_wal_merged_gens: Vec::new(),
+                    conflicting_mem_wal_compacted_sstables: Vec::new(),
                 })
             }
             Operation::Rewrite { groups, .. } => {
@@ -119,7 +121,7 @@ impl<'a> TransactionRebase<'a> {
                     initial_fragments,
                     modified_fragment_ids,
                     conflicting_frag_reuse_indices: Vec::new(),
-                    conflicting_mem_wal_merged_gens: Vec::new(),
+                    conflicting_mem_wal_compacted_sstables: Vec::new(),
                 })
             }
             Operation::DataReplacement { replacements } => {
@@ -134,7 +136,22 @@ impl<'a> TransactionRebase<'a> {
                     initial_fragments,
                     modified_fragment_ids,
                     conflicting_frag_reuse_indices: Vec::new(),
-                    conflicting_mem_wal_merged_gens: Vec::new(),
+                    conflicting_mem_wal_compacted_sstables: Vec::new(),
+                })
+            }
+            Operation::DataOverlay { groups } => {
+                let modified_fragment_ids =
+                    groups.iter().map(|g| g.fragment_id).collect::<HashSet<_>>();
+                let initial_fragments =
+                    initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids)
+                        .await;
+                Ok(Self {
+                    transaction,
+                    affected_rows,
+                    initial_fragments,
+                    modified_fragment_ids,
+                    conflicting_frag_reuse_indices: Vec::new(),
+                    conflicting_mem_wal_compacted_sstables: Vec::new(),
                 })
             }
             Operation::Merge { fragments, .. } => {
@@ -148,7 +165,7 @@ impl<'a> TransactionRebase<'a> {
                     initial_fragments,
                     modified_fragment_ids,
                     conflicting_frag_reuse_indices: Vec::new(),
-                    conflicting_mem_wal_merged_gens: Vec::new(),
+                    conflicting_mem_wal_compacted_sstables: Vec::new(),
                 })
             }
         }
@@ -219,6 +236,9 @@ impl<'a> TransactionRebase<'a> {
             Operation::DataReplacement { .. } => {
                 self.check_data_replacement_txn(other_transaction, other_version)
             }
+            Operation::DataOverlay { .. } => {
+                self.check_data_overlay_txn(other_transaction, other_version)
+            }
             Operation::Merge { .. } => self.check_merge_txn(other_transaction, other_version),
             Operation::Restore { .. } => self.check_restore_txn(other_transaction, other_version),
             Operation::ReserveFragments { .. } => {
@@ -251,6 +271,10 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Project { .. }
                 | Operation::Append { .. }
                 | Operation::UpdateConfig { .. }
+                // A concurrent overlay is inert against the rows we delete
+                // (deletions take precedence over overlays) and otherwise
+                // preserves physical offsets, so it never conflicts.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
                 Operation::Rewrite { groups, .. } => {
                     if groups
@@ -345,7 +369,9 @@ impl<'a> TransactionRebase<'a> {
     ) -> Result<()> {
         if let Operation::Update {
             inserted_rows_filter: self_inserted_rows_filter,
-            merged_generations: self_merged_generations,
+            compacted_sstables: self_compacted_sstables,
+            new_fragments: self_new_fragments,
+            update_mode: self_update_mode,
             ..
         } = &self.transaction.operation
         {
@@ -399,6 +425,49 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Clone { .. }
                 | Operation::UpdateConfig { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
+                Operation::DataOverlay { groups } => {
+                    // Our update recomputed rows from the pre-overlay base, so if
+                    // it commits over an overlay it would silently undo the
+                    // overlay's values for any cell it recomputed. A row-moving
+                    // update (RewriteRows) relocates the rows it touches out to
+                    // new fragments; only the rows it actually moved lose their
+                    // overlay, so we conflict only when the moved rows intersect
+                    // the overlay's coverage. An in-place column rewrite
+                    // (RewriteColumns) preserves offsets and just tombstones the
+                    // overlaid fields at build time, so it never conflicts.
+                    let moves_rows = !self_new_fragments.is_empty()
+                        && matches!(self_update_mode, Some(UpdateMode::RewriteRows) | None);
+                    if !moves_rows {
+                        return Ok(());
+                    }
+                    // `affected_rows` holds the physical offsets (per fragment)
+                    // this update moved. The overlay's coverage is in the same
+                    // physical-offset space, so we can intersect the two in
+                    // memory. Without affected rows we cannot be precise, so we
+                    // fall back to a fragment-granular conflict.
+                    for group in groups {
+                        if !self.modified_fragment_ids.contains(&group.fragment_id) {
+                            continue;
+                        }
+                        let Some(affected_rows) = self.affected_rows else {
+                            return Err(
+                                self.retryable_conflict_err(other_transaction, other_version)
+                            );
+                        };
+                        let Some(moved) =
+                            affected_rows.get_fragment_bitmap(group.fragment_id as u32)
+                        else {
+                            continue;
+                        };
+                        let coverage = overlay_group_coverage(group);
+                        if !(moved & &coverage).is_empty() {
+                            return Err(
+                                self.retryable_conflict_err(other_transaction, other_version)
+                            );
+                        }
+                    }
+                    Ok(())
+                }
                 Operation::Append { .. } => {
                     // If current transaction has primary key conflict detection,
                     // we can't safely commit against an Append because we don't
@@ -487,10 +556,10 @@ impl<'a> TransactionRebase<'a> {
                     Err(self.incompatible_conflict_err(other_transaction, other_version))
                 }
                 Operation::UpdateMemWalState {
-                    merged_generations: other_merged_generations,
-                } => self.check_merged_generations_conflict(
-                    other_merged_generations,
-                    self_merged_generations,
+                    compacted_sstables: other_compacted_sstables,
+                } => self.check_compacted_sstables_conflict(
+                    other_compacted_sstables,
+                    self_compacted_sstables,
                     other_transaction,
                     other_version,
                 ),
@@ -514,6 +583,10 @@ impl<'a> TransactionRebase<'a> {
             match &other_transaction.operation {
                 Operation::Append { .. }
                 | Operation::Clone { .. }
+                // An overlay committed after this index's version is newer than
+                // the index; the query path excludes its covered cells via the
+                // version gate, so the build does not conflict.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
                 Operation::CreateIndex {
                     new_indices: created_indices,
@@ -631,14 +704,14 @@ impl<'a> TransactionRebase<'a> {
                     Ok(())
                 }
                 Operation::UpdateMemWalState {
-                    merged_generations: other_merged_gens,
+                    compacted_sstables: other_compacted_sstables,
                 } => {
                     // CreateIndex of MemWalIndex is compatible with UpdateMemWalState
                     // as they can be rebased on each other
                     if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) {
-                        // Collect merged_generations from UpdateMemWalState for rebasing
-                        self.conflicting_mem_wal_merged_gens
-                            .extend(other_merged_gens.iter().cloned());
+                        // Collect compacted_sstables from UpdateMemWalState for rebasing
+                        self.conflicting_mem_wal_compacted_sstables
+                            .extend(other_compacted_sstables.iter().cloned());
                         Ok(())
                     } else {
                         Err(self.incompatible_conflict_err(other_transaction, other_version))
@@ -695,6 +768,20 @@ impl<'a> TransactionRebase<'a> {
                         Ok(())
                     }
                 }
+                Operation::DataOverlay { groups } => {
+                    // Rewriting a fragment changes its physical row addresses, so
+                    // an overlay addressed by physical offset on that fragment is
+                    // invalidated and must be re-applied against the new base.
+                    if groups
+                        .iter()
+                        .map(|g| g.fragment_id)
+                        .any(|id| self.modified_fragment_ids.contains(&id))
+                    {
+                        Err(self.retryable_conflict_err(other_transaction, other_version))
+                    } else {
+                        Ok(())
+                    }
+                }
                 Operation::Rewrite {
                     groups,
                     frag_reuse_index: committed_fri,
@@ -874,6 +961,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::Restore { .. }
             | Operation::ReserveFragments { .. }
@@ -907,7 +995,8 @@ impl<'a> TransactionRebase<'a> {
             | Operation::Merge { .. }
             | Operation::UpdateConfig { .. }
             | Operation::Clone { .. }
-            | Operation::DataReplacement { .. } => Ok(()),
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => Ok(()),
         }
     }
 
@@ -923,6 +1012,9 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::UpdateConfig { .. }
                 | Operation::ReserveFragments { .. }
                 | Operation::Project { .. }
+                // Both a column replacement and an overlay preserve physical row
+                // addresses; the overlay is newer and wins its covered cells.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
                 Operation::Merge { .. } => {
                     // Merge rewrites the whole fragment list; always conflict
@@ -1055,6 +1147,119 @@ impl<'a> TransactionRebase<'a> {
         }
     }
 
+    /// Conflict checks for our DataOverlay transaction against a concurrent one.
+    ///
+    /// Overlays are intentionally permissive (see the Data Overlay Files spec):
+    /// they stack with other overlays and tolerate appends, index builds, data
+    /// replacement, deletes, and in-place column rewrites (Update with
+    /// `RewriteColumns`), because overlay coverage is addressed by physical offset
+    /// and the version gate keeps indexes correct. A concurrent operation
+    /// conflicts when it takes precedence over the overlay for cells the overlay
+    /// covers, dropping the overlay's values: retryably when it rewrites the
+    /// physical layout of one of our fragments (Rewrite, Merge) or re-creates the
+    /// covered rows from the pre-overlay base (a row-moving Update — checked
+    /// row-by-row in `finish_data_overlay`), or removes an overlaid fragment
+    /// outright (a Delete / Update that drops the fragment); and incompatibly for
+    /// whole-dataset replacements (Overwrite / Restore) and MemWAL state updates
+    /// (UpdateMemWalState), which do not rebase against data operations.
+    fn check_data_overlay_txn(
+        &mut self,
+        other_transaction: &Transaction,
+        other_version: u64,
+    ) -> Result<()> {
+        match &other_transaction.operation {
+            Operation::Append { .. }
+            | Operation::CreateIndex { .. }
+            | Operation::ReserveFragments { .. }
+            | Operation::Project { .. }
+            | Operation::UpdateConfig { .. }
+            | Operation::UpdateBases { .. }
+            | Operation::Clone { .. }
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => Ok(()),
+            // A concurrent Delete only tombstones rows via a deletion vector,
+            // which preserves physical offsets; the overlay value for a deleted
+            // offset is simply inert. Conflict only if the whole overlaid
+            // fragment was removed, orphaning the overlay.
+            Operation::Delete {
+                deleted_fragment_ids,
+                ..
+            } => {
+                if deleted_fragment_ids
+                    .iter()
+                    .any(|id| self.modified_fragment_ids.contains(id))
+                {
+                    Err(self.retryable_conflict_err(other_transaction, other_version))
+                } else {
+                    Ok(())
+                }
+            }
+            // A concurrent Update that removed an overlaid fragment orphans the
+            // overlay outright — conflict. A row-moving update (RewriteRows)
+            // deletes the rows it touches and re-creates them in new fragments;
+            // the update took precedence and the re-created rows were computed
+            // from the pre-overlay base, so the overlay's values for those cells
+            // are lost. That is a per-row problem, not an offset one: only the
+            // moved rows are affected. Comparing the moved rows against the
+            // overlay's coverage needs the update's deletion vectors, so we mark
+            // the fragment here and verify row-by-row in `finish_data_overlay`.
+            // An in-place column rewrite (RewriteColumns) preserves rows and just
+            // tombstones the overlaid fields at build time, so it never conflicts.
+            Operation::Update {
+                removed_fragment_ids,
+                updated_fragments,
+                new_fragments,
+                update_mode,
+                ..
+            } => {
+                let removed_ours = removed_fragment_ids
+                    .iter()
+                    .any(|id| self.modified_fragment_ids.contains(id));
+                if removed_ours {
+                    return Err(self.retryable_conflict_err(other_transaction, other_version));
+                }
+                let moves_rows = !new_fragments.is_empty()
+                    && matches!(update_mode, Some(UpdateMode::RewriteRows) | None);
+                if moves_rows {
+                    for updated in updated_fragments {
+                        if let Some((_, needs_row_check)) =
+                            self.initial_fragments.get_mut(&updated.id)
+                        {
+                            *needs_row_check = true;
+                        }
+                    }
+                }
+                Ok(())
+            }
+            Operation::Rewrite { groups, .. } => {
+                // A rewrite (compaction / fold) of a fragment we are overlaying
+                // changes its physical row addresses, so our offsets would be
+                // invalid. Conflict only if it touches one of our fragments.
+                let touches_our_fragment = groups
+                    .iter()
+                    .flat_map(|g| g.old_fragments.iter())
+                    .any(|f| self.modified_fragment_ids.contains(&f.id));
+                if touches_our_fragment {
+                    Err(self.retryable_conflict_err(other_transaction, other_version))
+                } else {
+                    Ok(())
+                }
+            }
+            Operation::Merge { .. } => {
+                // Merge rewrites the whole fragment list; always conflict.
+                Err(self.retryable_conflict_err(other_transaction, other_version))
+            }
+            // Overwrite/Restore replace the dataset; UpdateMemWalState does not
+            // rebase against data operations (mirroring check_update_mem_wal_state_txn,
+            // which likewise treats a concurrent DataOverlay as incompatible).
+            Operation::Overwrite { .. }
+            | Operation::Restore { .. }
+            | Operation::UpdateMemWalState { .. } => {
+                Err(self.incompatible_conflict_err(other_transaction, other_version))
+            }
+        }
+    }
+
     fn check_merge_txn(
         &mut self,
         other_transaction: &Transaction,
@@ -1072,7 +1277,8 @@ impl<'a> TransactionRebase<'a> {
             | Operation::Delete { .. }
             | Operation::Rewrite { .. }
             | Operation::Merge { .. }
-            | Operation::DataReplacement { .. } => {
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => {
                 Err(self.retryable_conflict_err(other_transaction, other_version))
             }
             Operation::Overwrite { .. }
@@ -1096,6 +1302,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::Restore { .. }
             | Operation::ReserveFragments { .. }
@@ -1124,6 +1331,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::ReserveFragments { .. }
             | Operation::Update { .. }
@@ -1148,6 +1356,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::UpdateConfig { .. }
             | Operation::CreateIndex { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Rewrite { .. }
             | Operation::Clone { .. }
             | Operation::ReserveFragments { .. }
@@ -1212,6 +1421,7 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::CreateIndex { .. }
                 | Operation::Rewrite { .. }
                 | Operation::DataReplacement { .. }
+                | Operation::DataOverlay { .. }
                 | Operation::Merge { .. }
                 | Operation::Restore { .. }
                 | Operation::ReserveFragments { .. }
@@ -1231,44 +1441,44 @@ impl<'a> TransactionRebase<'a> {
         other_version: u64,
     ) -> Result<()> {
         if let Operation::UpdateMemWalState {
-            merged_generations: self_merged_generations,
+            compacted_sstables: self_compacted_sstables,
         } = &self.transaction.operation
         {
             match &other_transaction.operation {
                 Operation::UpdateMemWalState {
-                    merged_generations: other_merged_generations,
+                    compacted_sstables: other_compacted_sstables,
                 } => {
                     // Two UpdateMemWalState transactions conflict if they're updating
-                    // the same shard's merged_generation
-                    self.check_merged_generations_conflict(
-                        other_merged_generations,
-                        self_merged_generations,
+                    // the same shard's compacted SSTable
+                    self.check_compacted_sstables_conflict(
+                        other_compacted_sstables,
+                        self_compacted_sstables,
                         other_transaction,
                         other_version,
                     )
                 }
                 Operation::Update {
-                    merged_generations: other_merged_generations,
+                    compacted_sstables: other_compacted_sstables,
                     ..
                 } => {
-                    // Update transactions with merged_generations can conflict
-                    self.check_merged_generations_conflict(
-                        other_merged_generations,
-                        self_merged_generations,
+                    // Update transactions with compacted_sstables can conflict
+                    self.check_compacted_sstables_conflict(
+                        other_compacted_sstables,
+                        self_compacted_sstables,
                         other_transaction,
                         other_version,
                     )
                 }
                 Operation::CreateIndex { new_indices, .. } => {
-                    // Check if CreateIndex has a MemWalIndex with merged_generations
+                    // Check if CreateIndex has a MemWalIndex with compacted_sstables
                     if let Some(mem_wal_idx) = new_indices
                         .iter()
                         .find(|idx| idx.name == MEM_WAL_INDEX_NAME)
                     {
                         let details = load_mem_wal_index_details(mem_wal_idx.clone())?;
-                        self.check_merged_generations_conflict(
-                            &details.merged_generations,
-                            self_merged_generations,
+                        self.check_compacted_sstables_conflict(
+                            &details.compacted_sstables,
+                            self_compacted_sstables,
                             other_transaction,
                             other_version,
                         )
@@ -1284,6 +1494,7 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Overwrite { .. }
                 | Operation::Delete { .. }
                 | Operation::DataReplacement { .. }
+                | Operation::DataOverlay { .. }
                 | Operation::Merge { .. }
                 | Operation::Restore { .. }
                 | Operation::Clone { .. }
@@ -1339,10 +1550,10 @@ impl<'a> TransactionRebase<'a> {
         }
     }
 
-    fn check_merged_generations_conflict(
+    fn check_compacted_sstables_conflict(
         &self,
-        committed: &[MergedGeneration],
-        to_commit: &[MergedGeneration],
+        committed: &[CompactedSsTable],
+        to_commit: &[CompactedSsTable],
         other_transaction: &Transaction,
         other_version: u64,
     ) -> Result<()> {
@@ -1351,7 +1562,8 @@ impl<'a> TransactionRebase<'a> {
             for to_commit_mg in to_commit {
                 if committed_mg.shard_id == to_commit_mg.shard_id {
                     // Same shard being updated
-                    // If committed >= to_commit, data already merged or superseded - abort without retry
+                    // If committed >= to_commit, the SSTable is already compacted
+                    // or superseded, so abort without retry.
                     // If committed < to_commit, can retry with new state
                     if committed_mg.generation >= to_commit_mg.generation {
                         return Err(
@@ -1374,6 +1586,7 @@ impl<'a> TransactionRebase<'a> {
             }
             Operation::CreateIndex { .. } => self.finish_create_index(dataset).await,
             Operation::Rewrite { .. } => self.finish_rewrite(dataset).await,
+            Operation::DataOverlay { .. } => self.finish_data_overlay(dataset).await,
             Operation::Append { .. }
             | Operation::Overwrite { .. }
             | Operation::DataReplacement { .. }
@@ -1550,6 +1763,90 @@ impl<'a> TransactionRebase<'a> {
         }
     }
 
+    /// Verify no concurrent row-moving Update dropped the values of any cell
+    /// this overlay covers. `check_data_overlay_txn` flags (via the
+    /// `initial_fragments` needs-check bool) each overlaid fragment on which a
+    /// concurrent RewriteRows update relocated rows; here we read the deletion
+    /// vectors and conflict only when the moved rows intersect the overlay's
+    /// coverage.
+    ///
+    /// The moved rows are computed as the current deletion vector minus the
+    /// read-time one. In the rare case where both a concurrent Delete and a
+    /// concurrent Update touched the same flagged fragment, the Delete's rows are
+    /// also counted and may trigger an unnecessary retry — never data loss. Pure
+    /// concurrent deletes leave the fragment unflagged and are not examined here.
+    async fn finish_data_overlay(self, dataset: &Dataset) -> Result<Transaction> {
+        let fragments_to_check: HashSet<u64> = self
+            .initial_fragments
+            .iter()
+            .filter_map(|(id, (_, needs_check))| needs_check.then_some(*id))
+            .collect();
+        if fragments_to_check.is_empty() {
+            return Ok(Transaction {
+                read_version: dataset.manifest.version,
+                ..self.transaction
+            });
+        }
+
+        // Coverage (physical offsets, unioned across fields) per flagged fragment.
+        let Operation::DataOverlay { groups } = &self.transaction.operation else {
+            return Err(wrong_operation_err(&self.transaction.operation));
+        };
+        let mut coverage_by_fragment: HashMap<u64, RoaringBitmap> = HashMap::new();
+        for group in groups {
+            if !fragments_to_check.contains(&group.fragment_id) {
+                continue;
+            }
+            *coverage_by_fragment.entry(group.fragment_id).or_default() |=
+                overlay_group_coverage(group);
+        }
+
+        for (fragment_id, coverage) in coverage_by_fragment {
+            let Some(current_fragment) = dataset
+                .fragments()
+                .as_slice()
+                .iter()
+                .find(|f| f.id == fragment_id)
+            else {
+                // The fragment is gone entirely; the overlay is orphaned.
+                return Err(crate::Error::retryable_commit_conflict_source(
+                    dataset.manifest.version,
+                    format!(
+                        "This {} transaction was preempted: overlaid fragment {} was removed by a concurrent transaction. Please retry.",
+                        self.transaction.uuid, fragment_id
+                    )
+                    .into(),
+                ));
+            };
+            let current_deletions =
+                read_fragment_deletion_bitmap(dataset, current_fragment).await?;
+            let initial_deletions = match self.initial_fragments.get(&fragment_id) {
+                Some((initial_fragment, _)) => {
+                    read_fragment_deletion_bitmap(dataset, initial_fragment).await?
+                }
+                None => RoaringBitmap::new(),
+            };
+            let moved_rows = &current_deletions - &initial_deletions;
+            let conflicting = &moved_rows & &coverage;
+            if !conflicting.is_empty() {
+                let sample: Vec<u32> = conflicting.iter().take(5).collect();
+                return Err(crate::Error::retryable_commit_conflict_source(
+                    dataset.manifest.version,
+                    format!(
+                        "This {} transaction was preempted by a concurrent update that moved overlaid rows on fragment {} (offsets {:?}). Please retry.",
+                        self.transaction.uuid, fragment_id, sample.as_slice()
+                    )
+                    .into(),
+                ));
+            }
+        }
+
+        Ok(Transaction {
+            read_version: dataset.manifest.version,
+            ..self.transaction
+        })
+    }
+
     async fn finish_create_index(mut self, dataset: &Dataset) -> Result<Transaction> {
         if let Operation::CreateIndex {
             new_indices,
@@ -1608,7 +1905,7 @@ impl<'a> TransactionRebase<'a> {
             // Handle MEM_WAL_INDEX rebasing
             let has_mem_wal = new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME);
 
-            if has_mem_wal && !self.conflicting_mem_wal_merged_gens.is_empty() {
+            if has_mem_wal && !self.conflicting_mem_wal_compacted_sstables.is_empty() {
                 let pos = new_indices
                     .iter()
                     .position(|idx| idx.name == MEM_WAL_INDEX_NAME)
@@ -1617,19 +1914,19 @@ impl<'a> TransactionRebase<'a> {
                 let current_meta = new_indices.remove(pos);
                 let mut details = load_mem_wal_index_details(current_meta)?;
 
-                // Merge conflicting merged_generations - for each shard, keep higher generation
-                // We own self so we can consume conflicting_mem_wal_merged_gens directly
-                for new_mg in self.conflicting_mem_wal_merged_gens {
+                // Reconcile conflicting compacted_sstables by keeping each shard's higher generation.
+                // We own self so we can consume conflicting_mem_wal_compacted_sstables directly
+                for new_sstable in self.conflicting_mem_wal_compacted_sstables {
                     if let Some(existing) = details
-                        .merged_generations
+                        .compacted_sstables
                         .iter_mut()
-                        .find(|mg| mg.shard_id == new_mg.shard_id)
+                        .find(|sstable| sstable.shard_id == new_sstable.shard_id)
                     {
-                        if new_mg.generation > existing.generation {
-                            existing.generation = new_mg.generation;
+                        if new_sstable.generation > existing.generation {
+                            existing.generation = new_sstable.generation;
                         }
                     } else {
-                        details.merged_generations.push(new_mg);
+                        details.compacted_sstables.push(new_sstable);
                     }
                 }
 
@@ -1774,6 +2071,40 @@ async fn initial_fragments_for_rebase(
         .collect::<HashMap<_, _>>()
 }
 
+/// Read a fragment's deletion vector as a bitmap of physical offsets, or an
+/// empty bitmap when the fragment has no deletion file.
+async fn read_fragment_deletion_bitmap(
+    dataset: &Dataset,
+    fragment: &Fragment,
+) -> Result<RoaringBitmap> {
+    match &fragment.deletion_file {
+        Some(deletion_file) => {
+            let dv = read_dataset_deletion_file(dataset, fragment.id, deletion_file).await?;
+            Ok(RoaringBitmap::from(dv.as_ref()))
+        }
+        None => Ok(RoaringBitmap::new()),
+    }
+}
+
+/// The physical offsets a group's overlays cover, unioned across every overlay
+/// and every field. This is the set of cells whose values the overlay supplies,
+/// used to test whether a concurrent row-moving Update actually invalidates the
+/// overlay.
+fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap {
+    let mut union = RoaringBitmap::new();
+    for overlay in &group.overlays {
+        match &overlay.coverage {
+            OverlayCoverage::Shared(bitmap) => union |= bitmap.as_ref(),
+            OverlayCoverage::PerField(bitmaps) => {
+                for bitmap in bitmaps {
+                    union |= bitmap.as_ref();
+                }
+            }
+        }
+    }
+    union
+}
+
 fn wrong_operation_err(op: &Operation) -> Error {
     Error::internal(format!("function called against a wrong operation: {}", op))
 }
@@ -1884,7 +2215,7 @@ mod tests {
             removed_fragment_ids: vec![],
             new_fragments: vec![],
             fields_modified: vec![],
-            merged_generations: Vec::new(),
+            compacted_sstables: Vec::new(),
             fields_for_preserving_frag_bitmap: vec![],
             update_mode: None,
             inserted_rows_filter: None,
@@ -1897,7 +2228,7 @@ mod tests {
                 removed_fragment_ids: vec![2],
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -1913,7 +2244,7 @@ mod tests {
                 updated_fragments: vec![Fragment::new(4)],
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -2016,7 +2347,7 @@ mod tests {
                 removed_fragment_ids: vec![],
                 new_fragments: vec![sample_file.clone()],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -2032,7 +2363,7 @@ mod tests {
                 removed_fragment_ids: vec![],
                 new_fragments: vec![sample_file],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -2162,7 +2493,7 @@ mod tests {
                     removed_fragment_ids: vec![0],
                     new_fragments: vec![sample_file.clone()],
                     fields_modified: vec![],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: None,
                     inserted_rows_filter: None,
@@ -2176,7 +2507,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![sample_file.clone()],
                     fields_modified: vec![],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: None,
                     inserted_rows_filter: None,
@@ -2337,7 +2668,7 @@ mod tests {
                 updated_fragments: vec![fragment0.clone()],
                 new_fragments: vec![fragment2.clone()],
                 fields_modified: vec![0],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -2544,7 +2875,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![fragment2],
                     fields_modified: vec![0],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: None,
                     inserted_rows_filter: None,
@@ -2738,7 +3069,7 @@ mod tests {
                 modified_fragment_ids: modified_fragment_ids(operation).collect::<HashSet<_>>(),
                 affected_rows: None,
                 conflicting_frag_reuse_indices: Vec::new(),
-                conflicting_mem_wal_merged_gens: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
             };
 
             for (other, expected_conflict) in other_transactions.iter().zip(expected_conflicts) {
@@ -2781,6 +3112,442 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_data_overlay_conflicts() {
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use ConflictResult::*;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        // Our transaction overlays fragment 1.
+        let overlay_op = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        let update_removing = |removed_fragment_ids: Vec<u64>| Operation::Update {
+            removed_fragment_ids,
+            updated_fragments: vec![],
+            new_fragments: vec![],
+            fields_modified: vec![],
+            compacted_sstables: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: None,
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let delete = |updated: Vec<Fragment>, deleted: Vec<u64>| Operation::Delete {
+            updated_fragments: updated,
+            deleted_fragment_ids: deleted,
+            predicate: "x > 2".to_string(),
+        };
+        // A row-moving update (RewriteRows) relocates the updated rows into
+        // new_fragments; an in-place column rewrite (RewriteColumns) leaves rows
+        // where they are.
+        let update_moving = |updated: Vec<Fragment>, new: Vec<Fragment>| Operation::Update {
+            removed_fragment_ids: vec![],
+            updated_fragments: updated,
+            new_fragments: new,
+            fields_modified: vec![],
+            compacted_sstables: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteRows),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let update_rewrite_columns = |updated: Vec<Fragment>| Operation::Update {
+            removed_fragment_ids: vec![],
+            updated_fragments: updated,
+            new_fragments: vec![],
+            fields_modified: vec![0],
+            compacted_sstables: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteColumns),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let rewrite_of = |old: &Fragment| Operation::Rewrite {
+            groups: vec![RewriteGroup {
+                old_fragments: vec![old.clone()],
+                new_fragments: vec![],
+            }],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
+        };
+
+        let fragment0 = Fragment::new(0);
+        let fragment1 = Fragment::new(1);
+
+        // Each case is checked against our overlay on fragment 1.
+        let cases: Vec<(Operation, ConflictResult)> = vec![
+            // Permissive: preserves physical offsets / leaves fragment 1 in place.
+            (
+                Operation::Append {
+                    fragments: vec![fragment0.clone()],
+                },
+                Compatible,
+            ),
+            (
+                Operation::CreateIndex {
+                    new_indices: vec![],
+                    removed_indices: vec![],
+                },
+                Compatible,
+            ),
+            (
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(
+                        1,
+                        DataFile::new_legacy_from_fields("r.lance", vec![0], None),
+                    )],
+                },
+                Compatible,
+            ),
+            // Another overlay on the same fragment stacks rather than conflicts.
+            (overlay_op(1), Compatible),
+            // A Delete only tombstones rows (deletion vector) on fragment 1, and
+            // an in-place column rewrite preserves offsets, so both are compatible.
+            (delete(vec![fragment1.clone()], vec![]), Compatible),
+            (update_rewrite_columns(vec![fragment1.clone()]), Compatible),
+            (update_removing(vec![2]), Compatible),
+            // ...but removing our overlaid fragment 1 orphans the overlay -> conflict.
+            (delete(vec![], vec![1]), Retryable),
+            (update_removing(vec![1]), Retryable),
+            // A row-moving update re-creates the rows it touches from the
+            // pre-overlay base. Whether that actually drops any overlaid cell is
+            // a per-row question answered in `finish_data_overlay` (see
+            // test_data_overlay_finish_conflicts_with_row_moving_update), so the
+            // check itself defers rather than conflicting; a moving update on any
+            // fragment is compatible at this stage.
+            (
+                update_moving(vec![fragment1.clone()], vec![fragment0.clone()]),
+                Compatible,
+            ),
+            (
+                update_moving(vec![fragment0.clone()], vec![fragment0.clone()]),
+                Compatible,
+            ),
+            // Rewriting fragment 1 invalidates its physical offsets -> conflict;
+            // a rewrite of a different fragment does not.
+            (rewrite_of(&fragment1), Retryable),
+            (rewrite_of(&fragment0), Compatible),
+            // Merge rewrites the whole fragment list; Restore replaces the dataset.
+            (
+                Operation::Merge {
+                    fragments: vec![fragment1.clone()],
+                    schema: lance_core::datatypes::Schema::default(),
+                },
+                Retryable,
+            ),
+            (Operation::Restore { version: 1 }, NotCompatible),
+            // Overwrite/Restore replace the dataset, and UpdateMemWalState does
+            // not rebase against data operations — all hard conflicts.
+            (
+                Operation::Overwrite {
+                    fragments: vec![fragment0.clone()],
+                    schema: lance_core::datatypes::Schema::default(),
+                    config_upsert_values: None,
+                    initial_bases: None,
+                },
+                NotCompatible,
+            ),
+            (
+                Operation::UpdateMemWalState {
+                    compacted_sstables: vec![],
+                },
+                NotCompatible,
+            ),
+        ];
+
+        for (other, expected) in cases {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, overlay_op(1), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&overlay_op(1))
+                    .collect::<HashSet<_>>(),
+                affected_rows: None,
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            match expected {
+                Compatible => assert!(
+                    result.is_ok(),
+                    "overlay should be compatible with {other:?}, got {result:?}"
+                ),
+                Retryable => assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "overlay should retryably conflict with {other:?}, got {result:?}"
+                ),
+                NotCompatible => assert!(
+                    matches!(result, Err(Error::IncompatibleTransaction { .. })),
+                    "overlay should be incompatible with {other:?}, got {result:?}"
+                ),
+            }
+        }
+    }
+
+    #[test]
+    fn test_rewrite_conflicts_with_data_overlay() {
+        // Reverse direction of test_data_overlay_conflicts: our transaction is a
+        // Rewrite and a concurrent DataOverlay has already committed. A rewrite
+        // changes the physical row addresses of the fragments it touches, so an
+        // overlay on one of those fragments is invalidated (retryable); an
+        // overlay on any other fragment is unaffected.
+        use crate::dataset::transaction::DataOverlayGroup;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let overlay_on = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        // Our transaction rewrites fragment 1.
+        let rewrite_op = Operation::Rewrite {
+            groups: vec![RewriteGroup {
+                old_fragments: vec![Fragment::new(1)],
+                new_fragments: vec![],
+            }],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
+        };
+
+        for (other, expect_conflict) in [(overlay_on(1), true), (overlay_on(0), false)] {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, rewrite_op.clone(), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&rewrite_op).collect::<HashSet<_>>(),
+                affected_rows: None,
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            if expect_conflict {
+                assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "rewrite of fragment 1 should retryably conflict with {other:?}, got {result:?}"
+                );
+            } else {
+                assert!(
+                    result.is_ok(),
+                    "rewrite of fragment 1 should not conflict with {other:?}, got {result:?}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_update_conflicts_with_data_overlay() {
+        // Reverse direction of test_data_overlay_conflicts: our transaction is an
+        // Update and a concurrent DataOverlay has already committed. A row-moving
+        // update relocates the rows it touches, so an overlay on one of those
+        // fragments can no longer be applied (retryable); an overlay on any other
+        // fragment, or an in-place column rewrite, is compatible.
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let overlay_on = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        // Our update always touches fragment 1.
+        let update =
+            |update_mode: Option<UpdateMode>, new_fragments: Vec<Fragment>| Operation::Update {
+                removed_fragment_ids: vec![],
+                updated_fragments: vec![Fragment::new(1)],
+                new_fragments,
+                fields_modified: vec![0],
+                compacted_sstables: Vec::new(),
+                fields_for_preserving_frag_bitmap: vec![],
+                update_mode,
+                inserted_rows_filter: None,
+                updated_fragment_offsets: None,
+            };
+
+        // The overlay covers physical offset 0 of its fragment. Row addresses
+        // pack the fragment id in the high 32 bits and the offset in the low 32.
+        let rows_on = |fragment_id: u64, offsets: &[u32]| {
+            let mut map = RowAddrTreeMap::new();
+            map.insert_bitmap(
+                fragment_id as u32,
+                RoaringBitmap::from_iter(offsets.iter().copied()),
+            );
+            map
+        };
+
+        // (update, committed overlay, moved rows the update carries, expect conflict)
+        let cases = [
+            // Row-moving update whose moved rows include the overlaid cell -> the
+            // update would undo the overlay, so conflict.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                Some(rows_on(1, &[0])),
+                true,
+            ),
+            // ...but if the moved rows miss the overlaid cell, the overlay survives.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                Some(rows_on(1, &[5])),
+                false,
+            ),
+            // An overlay on a fragment the update did not touch is fine.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(0),
+                Some(rows_on(1, &[0])),
+                false,
+            ),
+            // An in-place column rewrite preserves rows -> compatible.
+            (
+                update(Some(UpdateMode::RewriteColumns), vec![]),
+                overlay_on(1),
+                Some(rows_on(1, &[0])),
+                false,
+            ),
+            // Without affected rows we cannot be precise, so a row-moving update
+            // on the overlaid fragment falls back to a conservative conflict.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                None,
+                true,
+            ),
+        ];
+
+        for (update_op, other, affected_rows, expect_conflict) in cases {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, update_op.clone(), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&update_op).collect::<HashSet<_>>(),
+                affected_rows: affected_rows.as_ref(),
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            if expect_conflict {
+                assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "update should retryably conflict with {other:?}, got {result:?}"
+                );
+            } else {
+                assert!(
+                    result.is_ok(),
+                    "update should be compatible with {other:?}, got {result:?}"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[rstest::rstest]
+    #[case::coverage_overlaps_moved_row(vec![0u32], true)]
+    #[case::coverage_disjoint_from_moved_row(vec![3u32], false)]
+    async fn test_data_overlay_finish_conflicts_with_row_moving_update(
+        #[case] coverage_offsets: Vec<u32>,
+        #[case] expect_conflict: bool,
+    ) {
+        // 5 rows in one fragment. A concurrent RewriteRows update moves row 0 out
+        // to a new fragment (deleting it from fragment 0). Our overlay on fragment
+        // 0 conflicts only when its coverage includes the moved row; the decision
+        // is made in finish, which reads the deletion vectors.
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let dataset = test_dataset(5, 1).await;
+        let mut fragment = dataset.fragments().as_slice()[0].clone();
+
+        let moved_fragment = Fragment::new(0)
+            .with_file(
+                "moved.lance",
+                vec![0],
+                vec![0],
+                &LanceFileVersion::Stable,
+                NonZero::new(10),
+            )
+            .with_physical_rows(1);
+        let update_op = Operation::Update {
+            updated_fragments: vec![apply_deletion(&[0], &mut fragment, &dataset).await],
+            removed_fragment_ids: vec![],
+            new_fragments: vec![moved_fragment],
+            fields_modified: vec![],
+            compacted_sstables: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteRows),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let update_txn = Transaction::new_from_version(dataset.manifest.version, update_op);
+
+        let overlay_op = Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id: 0,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(coverage_offsets)),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        let overlay_txn = Transaction::new_from_version(dataset.manifest.version, overlay_op);
+
+        // Commit the update so the latest dataset reflects the moved (deleted) row.
+        let latest_dataset = CommitBuilder::new(Arc::new(dataset.clone()))
+            .execute(update_txn.clone())
+            .await
+            .unwrap();
+
+        let mut rebase = TransactionRebase::try_new(&dataset, overlay_txn.clone(), None)
+            .await
+            .unwrap();
+        // The check defers the row-level decision to finish, flagging fragment 0.
+        rebase.check_txn(&update_txn, 1).unwrap();
+        assert_eq!(
+            rebase
+                .initial_fragments
+                .iter()
+                .map(|(id, (_, needs_check))| (*id, *needs_check))
+                .collect::<Vec<_>>(),
+            vec![(0, true)],
+        );
+
+        let res = rebase.finish(&latest_dataset).await;
+        if expect_conflict {
+            assert!(
+                matches!(res, Err(crate::Error::RetryableCommitConflict { .. })),
+                "overlay covering the moved row should conflict, got {res:?}"
+            );
+        } else {
+            assert!(
+                res.is_ok(),
+                "overlay disjoint from the moved row should succeed, got {res:?}"
+            );
+        }
+    }
+
     #[test]
     fn test_create_index_conflicts_only_on_same_name() {
         let index0 = IndexMetadata {
@@ -2815,7 +3582,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let same_name = Transaction::new(
@@ -2869,7 +3636,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
         let different_name_result = rebase.check_txn(&different_name, 1);
         assert!(
@@ -3067,7 +3834,7 @@ mod tests {
                 removed_fragment_ids: vec![],
                 new_fragments: vec![],
                 fields_modified: vec![],
-                merged_generations: Vec::new(),
+                compacted_sstables: Vec::new(),
                 fields_for_preserving_frag_bitmap: vec![],
                 update_mode: None,
                 inserted_rows_filter: None,
@@ -3255,6 +4022,7 @@ mod tests {
             Operation::DataReplacement { replacements } => {
                 Box::new(replacements.iter().map(|r| r.0))
             }
+            Operation::DataOverlay { groups } => Box::new(groups.iter().map(|g| g.fragment_id)),
         }
     }
 
@@ -3359,7 +4127,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![],
                     fields_modified: vec![2],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: Some(RewriteColumns),
                     inserted_rows_filter: None,
@@ -3378,7 +4146,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![Fragment::new(5)],
                     fields_modified: vec![2],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: Some(RewriteColumns),
                     inserted_rows_filter: None,
@@ -3396,7 +4164,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![],
                     fields_modified: vec![1],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: Some(RewriteColumns),
                     inserted_rows_filter: None,
@@ -3414,7 +4182,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![Fragment::new(5)],
                     fields_modified: vec![],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: Some(RewriteRows),
                     inserted_rows_filter: None,
@@ -3432,7 +4200,7 @@ mod tests {
                     removed_fragment_ids: vec![0],
                     new_fragments: vec![],
                     fields_modified: vec![],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: None,
                     inserted_rows_filter: None,
@@ -3450,7 +4218,7 @@ mod tests {
                     removed_fragment_ids: vec![],
                     new_fragments: vec![Fragment::new(5)],
                     fields_modified: vec![],
-                    merged_generations: Vec::new(),
+                    compacted_sstables: Vec::new(),
                     fields_for_preserving_frag_bitmap: vec![],
                     update_mode: Some(RewriteRows),
                     inserted_rows_filter: None,
@@ -3506,7 +4274,7 @@ mod tests {
                 modified_fragment_ids: modified_fragment_ids(&op1).collect::<HashSet<_>>(),
                 affected_rows: None,
                 conflicting_frag_reuse_indices: Vec::new(),
-                conflicting_mem_wal_merged_gens: Vec::new(),
+                conflicting_mem_wal_compacted_sstables: Vec::new(),
             };
 
             let result = rebase.check_txn(&txn2, 1);
@@ -3542,7 +4310,7 @@ mod tests {
     }
 
     #[test]
-    fn test_merged_generations_conflict_lower_generation_fails() {
+    fn test_compacted_sstables_conflict_lower_generation_fails() {
         // Test: committed generation >= to_commit generation should be incompatible (no retry)
         let shard = Uuid::new_v4();
 
@@ -3550,7 +4318,7 @@ mod tests {
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -3558,7 +4326,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -3569,7 +4337,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result = rebase.check_txn(&committed_txn, 1);
@@ -3581,14 +4349,14 @@ mod tests {
     }
 
     #[test]
-    fn test_merged_generations_conflict_equal_generation_fails() {
+    fn test_compacted_sstables_conflict_equal_generation_fails() {
         // Test: committed generation == to_commit generation should be incompatible (no retry)
         let shard = Uuid::new_v4();
 
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -3596,7 +4364,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -3607,7 +4375,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result = rebase.check_txn(&committed_txn, 1);
@@ -3619,7 +4387,7 @@ mod tests {
     }
 
     #[test]
-    fn test_merged_generations_conflict_higher_generation_retryable() {
+    fn test_compacted_sstables_conflict_higher_generation_retryable() {
         // Test: committed generation < to_commit generation should be retryable
         let shard = Uuid::new_v4();
 
@@ -3627,7 +4395,7 @@ mod tests {
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -3635,7 +4403,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -3646,7 +4414,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result = rebase.check_txn(&committed_txn, 1);
@@ -3658,7 +4426,7 @@ mod tests {
     }
 
     #[test]
-    fn test_merged_generations_different_shards_ok() {
+    fn test_compacted_sstables_different_shards_ok() {
         // Test: different shards should not conflict
         let shard1 = Uuid::new_v4();
         let shard2 = Uuid::new_v4();
@@ -3666,7 +4434,7 @@ mod tests {
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard1, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard1, 10)],
             },
             None,
         );
@@ -3674,7 +4442,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard2, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard2, 5)],
             },
             None,
         );
@@ -3685,7 +4453,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result = rebase.check_txn(&committed_txn, 1);
@@ -3697,15 +4465,15 @@ mod tests {
     }
 
     #[test]
-    fn test_update_mem_wal_state_vs_create_index_with_merged_generations() {
+    fn test_update_mem_wal_state_vs_create_index_with_compacted_sstables() {
         use crate::index::mem_wal::new_mem_wal_index_meta;
         use lance_index::mem_wal::MemWalIndexDetails;
 
         let shard = Uuid::new_v4();
 
-        // Create a MemWalIndex with merged_generations
+        // Create a MemWalIndex with compacted_sstables
         let details = MemWalIndexDetails {
-            merged_generations: vec![MergedGeneration::new(shard, 10)],
+            compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             ..Default::default()
         };
         let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap();
@@ -3724,7 +4492,7 @@ mod tests {
         let to_commit_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 5)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 5)],
             },
             None,
         );
@@ -3735,7 +4503,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result = rebase.check_txn(&committed_txn, 1);
@@ -3749,7 +4517,7 @@ mod tests {
         let to_commit_txn_higher = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 15)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 15)],
             },
             None,
         );
@@ -3760,7 +4528,7 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         let result_higher = rebase_higher.check_txn(&committed_txn, 1);
@@ -3778,7 +4546,7 @@ mod tests {
 
         let shard = Uuid::new_v4();
 
-        // CreateIndex with MemWalIndex (no merged_generations initially)
+        // CreateIndex with MemWalIndex (no compacted_sstables initially)
         let details = MemWalIndexDetails::default();
         let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap();
 
@@ -3795,7 +4563,7 @@ mod tests {
         let committed_txn = Transaction::new(
             0,
             Operation::UpdateMemWalState {
-                merged_generations: vec![MergedGeneration::new(shard, 10)],
+                compacted_sstables: vec![CompactedSsTable::new(shard, 10)],
             },
             None,
         );
@@ -3806,11 +4574,11 @@ mod tests {
             modified_fragment_ids: HashSet::new(),
             affected_rows: None,
             conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+            conflicting_mem_wal_compacted_sstables: Vec::new(),
         };
 
         // CreateIndex of MemWalIndex should be compatible with UpdateMemWalState
-        // and should collect the merged_generations for rebasing
+        // and should collect the compacted_sstables for rebasing
         let result = rebase.check_txn(&committed_txn, 1);
         assert!(
             result.is_ok(),
@@ -3818,10 +4586,16 @@ mod tests {
             result
         );
 
-        // Verify that merged_generations were collected
-        assert_eq!(rebase.conflicting_mem_wal_merged_gens.len(), 1);
-        assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].shard_id, shard);
-        assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].generation, 10);
+        // Verify that compacted_sstables were collected
+        assert_eq!(rebase.conflicting_mem_wal_compacted_sstables.len(), 1);
+        assert_eq!(
+            rebase.conflicting_mem_wal_compacted_sstables[0].shard_id,
+            shard
+        );
+        assert_eq!(
+            rebase.conflicting_mem_wal_compacted_sstables[0].generation,
+            10
+        );
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/io/commit/external_manifest.rs b/rust/lance/src/io/commit/external_manifest.rs
index 850d10f9a23..9efecbda5e9 100644
--- a/rust/lance/src/io/commit/external_manifest.rs
+++ b/rust/lance/src/io/commit/external_manifest.rs
@@ -14,10 +14,11 @@ mod test {
     use futures::stream::BoxStream;
     use futures::{StreamExt, TryStreamExt, future::join_all};
     use lance_core::{Error, Result};
+    use lance_io::object_store::ObjectStore;
     use lance_table::io::commit::external_manifest::{
         ExternalManifestCommitHandler, ExternalManifestStore,
     };
-    use lance_table::io::commit::{CommitHandler, ManifestNamingScheme};
+    use lance_table::io::commit::{CommitHandler, ManifestLocation, ManifestNamingScheme};
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
     use object_store::memory::InMemory;
     use object_store::{
@@ -125,6 +126,76 @@ mod test {
         }
     }
 
+    #[derive(Debug)]
+    struct StaticExternalManifestStore {
+        location: ManifestLocation,
+        verify_store: Option<Arc<dyn OSObjectStore>>,
+    }
+
+    #[async_trait]
+    impl ExternalManifestStore for StaticExternalManifestStore {
+        async fn get(&self, _uri: &str, version: u64) -> Result<String> {
+            if version == self.location.version {
+                Ok(self.location.path.to_string())
+            } else {
+                Err(Error::not_found(format!("version {}", version)))
+            }
+        }
+
+        async fn get_manifest_location(
+            &self,
+            _base_uri: &str,
+            version: u64,
+        ) -> Result<ManifestLocation> {
+            if version == self.location.version {
+                Ok(self.location.clone())
+            } else {
+                Err(Error::not_found(format!("version {}", version)))
+            }
+        }
+
+        async fn get_latest_version(&self, _uri: &str) -> Result<Option<(u64, String)>> {
+            Ok(Some((
+                self.location.version,
+                self.location.path.to_string(),
+            )))
+        }
+
+        async fn get_latest_manifest_location(
+            &self,
+            _base_uri: &str,
+        ) -> Result<Option<ManifestLocation>> {
+            Ok(Some(self.location.clone()))
+        }
+
+        async fn put_if_not_exists(
+            &self,
+            _uri: &str,
+            _version: u64,
+            _path: &str,
+            _size: u64,
+            _e_tag: Option<String>,
+        ) -> Result<()> {
+            Ok(())
+        }
+
+        async fn put_if_exists(
+            &self,
+            _uri: &str,
+            _version: u64,
+            path: &str,
+            size: u64,
+            e_tag: Option<String>,
+        ) -> Result<()> {
+            if let Some(store) = &self.verify_store {
+                let final_meta = store.head(&Path::from(path)).await?;
+                assert_eq!(size, final_meta.size);
+                assert_eq!(e_tag, final_meta.e_tag);
+            }
+            Ok(())
+        }
+    }
+
     fn read_params(handler: Arc<dyn CommitHandler>) -> ReadParams {
         ReadParams {
             commit_handler: Some(handler),
@@ -139,6 +210,242 @@ mod test {
         }
     }
 
+    #[tokio::test]
+    async fn finalized_external_manifest_location_is_head_checked() {
+        let sleepy_store = SleepyExternalManifestStore::new();
+        let inner_store = sleepy_store.store.clone();
+        let handler = ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(sleepy_store),
+        };
+        let object_store = ObjectStore::memory();
+        let base_path = Path::from("repro");
+        let version = 7;
+        let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version);
+        let body = b"manifest body";
+
+        object_store
+            .inner
+            .put(&final_path, PutPayload::from_static(body))
+            .await
+            .expect("seed finalized manifest");
+        inner_store
+            .lock()
+            .await
+            .insert((base_path.to_string(), version), final_path.to_string());
+
+        let location = handler
+            .resolve_latest_location(&base_path, &object_store)
+            .await
+            .expect("resolve latest finalized manifest");
+
+        assert_eq!(location.path, final_path);
+        assert_eq!(location.size, Some(body.len() as u64));
+        assert_eq!(location.naming_scheme, ManifestNamingScheme::V2);
+    }
+
+    #[tokio::test]
+    async fn finalized_external_manifest_location_falls_back_to_v1() {
+        let sleepy_store = SleepyExternalManifestStore::new();
+        let inner_store = sleepy_store.store.clone();
+        let handler = ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(sleepy_store),
+        };
+        let object_store = ObjectStore::memory();
+        let base_path = Path::from("repro");
+        let version = 7;
+        let missing_v2_path = ManifestNamingScheme::V2.manifest_path(&base_path, version);
+        let v1_path = ManifestNamingScheme::V1.manifest_path(&base_path, version);
+
+        object_store
+            .inner
+            .put(&v1_path, PutPayload::from_static(b"v1 manifest body"))
+            .await
+            .expect("seed V1 manifest");
+        inner_store.lock().await.insert(
+            (base_path.to_string(), version),
+            missing_v2_path.to_string(),
+        );
+
+        let latest_location = handler
+            .resolve_latest_location(&base_path, &object_store)
+            .await
+            .expect("resolve latest should fall back to V1");
+        assert_eq!(latest_location.path, v1_path);
+        assert_eq!(latest_location.naming_scheme, ManifestNamingScheme::V1);
+
+        let version_location = handler
+            .resolve_version_location(&base_path, version, object_store.inner.as_ref())
+            .await
+            .expect("resolve version should fall back to V1");
+        assert_eq!(version_location.path, v1_path);
+        assert_eq!(version_location.naming_scheme, ManifestNamingScheme::V1);
+    }
+
+    #[tokio::test]
+    async fn finalized_external_manifest_location_rejects_size_mismatch() {
+        let object_store = ObjectStore::memory();
+        let base_path = Path::from("repro");
+        let version = 7;
+        let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version);
+        let body = b"manifest body";
+
+        object_store
+            .inner
+            .put(&final_path, PutPayload::from_static(body))
+            .await
+            .expect("seed finalized manifest");
+        let handler = ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(StaticExternalManifestStore {
+                location: ManifestLocation {
+                    version,
+                    path: final_path,
+                    size: Some(body.len() as u64 + 1),
+                    naming_scheme: ManifestNamingScheme::V2,
+                    e_tag: None,
+                },
+                verify_store: None,
+            }),
+        };
+
+        let err = handler
+            .resolve_latest_location(&base_path, &object_store)
+            .await
+            .expect_err("stale external manifest size should be rejected");
+        assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}");
+        assert!(err.to_string().contains("Manifest size mismatch"), "{err}");
+    }
+
+    #[tokio::test]
+    async fn finalized_external_manifest_location_rejects_etag_mismatch() {
+        let object_store = ObjectStore::memory();
+        let base_path = Path::from("repro");
+        let version = 7;
+        let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version);
+        let body = b"manifest body";
+
+        object_store
+            .inner
+            .put(&final_path, PutPayload::from_static(body))
+            .await
+            .expect("seed finalized manifest");
+        let handler = ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(StaticExternalManifestStore {
+                location: ManifestLocation {
+                    version,
+                    path: final_path,
+                    size: Some(body.len() as u64),
+                    naming_scheme: ManifestNamingScheme::V2,
+                    e_tag: Some("stale-etag".to_string()),
+                },
+                verify_store: None,
+            }),
+        };
+
+        let err = handler
+            .resolve_latest_location(&base_path, &object_store)
+            .await
+            .expect_err("stale external manifest e_tag should be rejected");
+        assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}");
+        assert!(err.to_string().contains("Manifest e_tag mismatch"), "{err}");
+    }
+
+    #[tokio::test]
+    async fn external_manifest_store_put_records_destination_metadata() {
+        let object_store: Arc<dyn OSObjectStore> = Arc::new(InMemory::new());
+        let base_path = Path::from("repro");
+        let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd");
+        object_store
+            .put(&staging_path, PutPayload::from_static(b"manifest body"))
+            .await
+            .expect("seed staging manifest");
+        let staging_meta = object_store
+            .head(&staging_path)
+            .await
+            .expect("read staging metadata");
+
+        let external_store = StaticExternalManifestStore {
+            location: ManifestLocation {
+                version: 1,
+                path: staging_path.clone(),
+                size: Some(staging_meta.size),
+                naming_scheme: ManifestNamingScheme::V2,
+                e_tag: staging_meta.e_tag.clone(),
+            },
+            verify_store: Some(object_store.clone()),
+        };
+        let location = external_store
+            .put(
+                &base_path,
+                1,
+                &staging_path,
+                staging_meta.size,
+                staging_meta.e_tag.clone(),
+                object_store.as_ref(),
+                ManifestNamingScheme::V2,
+            )
+            .await
+            .expect("finalize manifest");
+        let final_meta = object_store
+            .head(&location.path)
+            .await
+            .expect("read finalized metadata");
+
+        assert_ne!(
+            staging_meta.e_tag, final_meta.e_tag,
+            "test store must assign a new ETag to the copied object"
+        );
+        assert_eq!(location.size, Some(final_meta.size));
+        assert_eq!(location.e_tag, final_meta.e_tag);
+    }
+
+    #[tokio::test]
+    async fn external_manifest_handler_finalize_records_destination_metadata() {
+        let object_store = ObjectStore::memory();
+        let base_path = Path::from("repro");
+        let version = 1;
+        let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd");
+        object_store
+            .inner
+            .put(&staging_path, PutPayload::from_static(b"manifest body"))
+            .await
+            .expect("seed staging manifest");
+        let staging_meta = object_store
+            .inner
+            .head(&staging_path)
+            .await
+            .expect("read staging metadata");
+
+        let handler = ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(StaticExternalManifestStore {
+                location: ManifestLocation {
+                    version,
+                    path: staging_path,
+                    size: Some(staging_meta.size),
+                    naming_scheme: ManifestNamingScheme::V2,
+                    e_tag: staging_meta.e_tag.clone(),
+                },
+                verify_store: Some(object_store.inner.clone()),
+            }),
+        };
+
+        let location = handler
+            .resolve_version_location(&base_path, version, object_store.inner.as_ref())
+            .await
+            .expect("finalize manifest");
+        let final_meta = object_store
+            .inner
+            .head(&location.path)
+            .await
+            .expect("read finalized metadata");
+
+        assert_ne!(
+            staging_meta.e_tag, final_meta.e_tag,
+            "test store must assign a new ETag to the copied object"
+        );
+        assert_eq!(location.size, Some(final_meta.size));
+        assert_eq!(location.e_tag, final_meta.e_tag);
+    }
+
     #[tokio::test]
     async fn test_dataset_can_onboard_external_store() {
         // First write a dataset WITHOUT external store
diff --git a/rust/lance/src/io/commit/s3_test.rs b/rust/lance/src/io/commit/s3_test.rs
index b5b1a09c776..6a714e8e576 100644
--- a/rust/lance/src/io/commit/s3_test.rs
+++ b/rust/lance/src/io/commit/s3_test.rs
@@ -304,9 +304,12 @@ async fn test_ddb_open_iops() {
     //    * write staged file
     //    * copy to final file
     //    * delete staged file
+    // Commit: 2 read IOPs:
+    // * list versions before creating the dataset
+    // * HEAD the finalized manifest to record destination metadata after copy
     let io_stats = committed_ds.object_store.as_ref().io_stats_incremental();
     assert_io_eq!(io_stats, write_iops, 4);
-    assert_io_eq!(io_stats, read_iops, 1);
+    assert_io_eq!(io_stats, read_iops, 2);
 
     let dataset = DatasetBuilder::from_uri(&uri)
         .with_read_params(ReadParams {
@@ -317,9 +320,10 @@ async fn test_ddb_open_iops() {
         .await
         .unwrap();
     let io_stats = dataset.object_store.as_ref().io_stats_incremental();
-    // Open dataset can be read with 1 IOP, just to read the manifest.
-    // Looking up latest manifest is handled in dynamodb.
-    assert_io_eq!(io_stats, read_iops, 1);
+    // Open dataset can be read with 2 IOPs: HEAD verifies that the
+    // finalized path returned by DynamoDB still exists, then the manifest
+    // itself is read. Looking up the latest manifest is handled in DynamoDB.
+    assert_io_eq!(io_stats, read_iops, 2);
     assert_io_eq!(io_stats, write_iops, 0);
 
     // Append
@@ -334,14 +338,17 @@ async fn test_ddb_open_iops() {
     let io_stats = dataset.object_store.as_ref().io_stats_incremental();
     // Append: 5 IOPS: data file, transaction file, 3x manifest file
     assert_io_eq!(io_stats, write_iops, 5);
+    // Append reads once to list versions and once to HEAD the finalized manifest
+    // so the destination metadata can be recorded after copy.
     // TODO: we can reduce this by implementing a specialized CommitHandler::list_manifest_locations()
     // for the DDB commit handler.
-    assert_io_eq!(io_stats, read_iops, 1);
+    assert_io_eq!(io_stats, read_iops, 2);
 
     // Checkout original version
     dataset.checkout_version(1).await.unwrap();
     let io_stats = dataset.object_store.as_ref().io_stats_incremental();
-    // Checkout: 1 IOPS: manifest file
+    // Checkout: 1 read IOP. Version resolution HEAD-checks the finalized path,
+    // while the manifest body is served from the Session metadata cache.
     assert_io_eq!(io_stats, read_iops, 1);
     assert_io_eq!(io_stats, write_iops, 0);
 }
diff --git a/rust/lance/src/io/exec/count_from_mask.rs b/rust/lance/src/io/exec/count_from_mask.rs
index 0b7aeb11111..e2f7f00efb5 100644
--- a/rust/lance/src/io/exec/count_from_mask.rs
+++ b/rust/lance/src/io/exec/count_from_mask.rs
@@ -395,10 +395,6 @@ impl ExecutionPlan for CountFromMaskExec {
         "CountFromMaskExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
@@ -463,11 +459,11 @@ impl ExecutionPlan for CountFromMaskExec {
     fn partition_statistics(
         &self,
         _partition: Option<usize>,
-    ) -> datafusion::error::Result<datafusion::physical_plan::Statistics> {
-        Ok(datafusion::physical_plan::Statistics {
+    ) -> datafusion::error::Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(datafusion::physical_plan::Statistics {
             num_rows: datafusion::common::stats::Precision::Exact(1),
             ..datafusion::physical_plan::Statistics::new_unknown(&self.schema)
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -494,7 +490,6 @@ mod tests {
     use datafusion::logical_expr::lit;
     use datafusion::physical_expr::execution_props::ExecutionProps;
     use datafusion::physical_plan::ExecutionPlan;
-    use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter;
     use datafusion::scalar::ScalarValue;
     use futures::TryStreamExt;
     use lance_core::utils::tempfile::TempStrDir;
@@ -512,8 +507,13 @@ mod tests {
     use crate::index::DatasetIndexExt;
     use crate::io::exec::scalar_index::ScalarIndexExec;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
+    #[allow(deprecated)]
+    use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter;
 
     /// Build an `AggregateFunctionExpr` matching `COUNT(*)`.
+    // TODO(datafusion-54): migrate off the deprecated
+    // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder.
+    #[allow(deprecated)]
     fn count_star_expr(input_schema: &SchemaRef) -> Arc<AggregateFunctionExpr> {
         let expr = functions_aggregate::count::count(lit(1));
         let df_schema = DFSchema::try_from(input_schema.as_ref().clone()).unwrap();
diff --git a/rust/lance/src/io/exec/count_pushdown.rs b/rust/lance/src/io/exec/count_pushdown.rs
index d5d90b5881a..6763db313aa 100644
--- a/rust/lance/src/io/exec/count_pushdown.rs
+++ b/rust/lance/src/io/exec/count_pushdown.rs
@@ -83,7 +83,7 @@ impl PhysicalOptimizerRule for CountPushdown {
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
         Ok(plan
             .transform_down(|plan| {
-                let Some(agg) = plan.as_any().downcast_ref::<AggregateExec>() else {
+                let Some(agg) = plan.downcast_ref::<AggregateExec>() else {
                     return Ok(Transformed::no(plan));
                 };
                 if let Some(rewritten) = try_rewrite(agg)? {
@@ -147,6 +147,11 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult<Option<Arc<dyn ExecutionPlan>>>
     };
 
     let options = filtered_read.options();
+    // We don't currently support count pushdown when the row selector
+    // is a row stream.
+    if filtered_read.row_stream_input().is_some() {
+        return Ok(None);
+    }
     // A refine filter is a residual the index couldn't fully evaluate — it
     // needs column data to apply, which we can't.
     if options.refine_filter.is_some() {
@@ -202,15 +207,12 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult<Option<Arc<dyn ExecutionPlan>>>
     let index_coverage = match &prefilter_input {
         None => None,
         Some(input) => {
-            let scalar_exec = input
-                .as_any()
-                .downcast_ref::<ScalarIndexExec>()
-                .ok_or_else(|| {
-                    datafusion::error::DataFusionError::Internal(
-                        "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec"
-                            .to_string(),
-                    )
-                })?;
+            let scalar_exec = input.downcast_ref::<ScalarIndexExec>().ok_or_else(|| {
+                datafusion::error::DataFusionError::Internal(
+                    "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec"
+                        .to_string(),
+                )
+            })?;
             if scalar_exec.expr().needs_recheck() {
                 return Ok(None);
             }
@@ -367,20 +369,21 @@ fn build_scan_branch(
 fn strip_row_preserving_wrappers(plan: &Arc<dyn ExecutionPlan>) -> Option<&FilteredReadExec> {
     let mut current: &dyn ExecutionPlan = plan.as_ref();
     loop {
-        if let Some(filtered_read) = current.as_any().downcast_ref::<FilteredReadExec>() {
+        if let Some(filtered_read) = current.downcast_ref::<FilteredReadExec>() {
             return Some(filtered_read);
         }
         let next: &Arc<dyn ExecutionPlan> =
-            if let Some(inner) = current.as_any().downcast_ref::<RepartitionExec>() {
+            if let Some(inner) = current.downcast_ref::<RepartitionExec>() {
                 inner.input()
             } else if let Some(inner) = {
                 #[allow(deprecated)]
-                current.as_any().downcast_ref::<CoalesceBatchesExec>()
+                current.downcast_ref::<CoalesceBatchesExec>()
             } {
                 inner.input()
-            } else if let Some(inner) = current.as_any().downcast_ref::<CoalescePartitionsExec>() {
+            } else if let Some(inner) = current.downcast_ref::<CoalescePartitionsExec>() {
                 inner.input()
-            } else if let Some(proj) = current.as_any().downcast_ref::<ProjectionExec>() {
+            } else {
+                let proj = current.downcast_ref::<ProjectionExec>()?;
                 // Only walk through projections that are row-preserving: every
                 // output expression is a direct column reference back to the
                 // input. (Empty projections trivially qualify — DataFusion uses
@@ -390,7 +393,6 @@ fn strip_row_preserving_wrappers(plan: &Arc<dyn ExecutionPlan>) -> Option<&Filte
                 let identity = proj.expr().iter().all(|projection_expr| {
                     projection_expr
                         .expr
-                        .as_any()
                         .downcast_ref::<Column>()
                         .is_some_and(|c| c.name() == input_schema.field(c.index()).name())
                 });
@@ -398,8 +400,6 @@ fn strip_row_preserving_wrappers(plan: &Arc<dyn ExecutionPlan>) -> Option<&Filte
                     return None;
                 }
                 proj.input()
-            } else {
-                return None;
             };
         current = next.as_ref();
     }
@@ -434,7 +434,7 @@ fn is_count_star(af: &Arc<AggregateFunctionExpr>) -> bool {
     if args.len() != 1 {
         return false;
     }
-    let Some(lit) = args[0].as_any().downcast_ref::<Literal>() else {
+    let Some(lit) = args[0].downcast_ref::<Literal>() else {
         return false;
     };
     // `COUNT(NULL)` would always return 0; rule it out so we don't accidentally
@@ -497,7 +497,7 @@ mod tests {
     fn plan_contains_pushdown(plan: &Arc<dyn ExecutionPlan>) -> bool {
         let mut found = false;
         plan.apply(|node| {
-            if node.as_any().is::<CountFromMaskExec>() {
+            if node.is::<CountFromMaskExec>() {
                 found = true;
                 Ok(TreeNodeRecursion::Stop)
             } else {
@@ -511,7 +511,7 @@ mod tests {
     fn plan_contains_union(plan: &Arc<dyn ExecutionPlan>) -> bool {
         let mut found = false;
         plan.apply(|node| {
-            if node.as_any().is::<UnionExec>() {
+            if node.is::<UnionExec>() {
                 found = true;
                 Ok(TreeNodeRecursion::Stop)
             } else {
diff --git a/rust/lance/src/io/exec/filter.rs b/rust/lance/src/io/exec/filter.rs
index 3a36f8d6712..71f1a5b2b4a 100644
--- a/rust/lance/src/io/exec/filter.rs
+++ b/rust/lance/src/io/exec/filter.rs
@@ -48,10 +48,6 @@ impl ExecutionPlan for LanceFilterExec {
         "LanceFilterExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.filter.properties()
     }
@@ -71,7 +67,6 @@ impl ExecutionPlan for LanceFilterExec {
         // Rewrap the result in a LanceFilterExec to preserve the logical expression
         let new_filter_plan = self.filter.clone().with_new_children(children)?;
         let new_filter = new_filter_plan
-            .as_any()
             .downcast_ref::<FilterExec>()
             .expect("FilterExec::with_new_children should return FilterExec")
             .clone();
@@ -93,7 +88,7 @@ impl ExecutionPlan for LanceFilterExec {
         self.filter.metrics()
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         self.filter.partition_statistics(partition)
     }
 
diff --git a/rust/lance/src/io/exec/filtered_read.rs b/rust/lance/src/io/exec/filtered_read.rs
index 059fa4e2b36..dddf18df4bd 100644
--- a/rust/lance/src/io/exec/filtered_read.rs
+++ b/rust/lance/src/io/exec/filtered_read.rs
@@ -1,20 +1,24 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
-use std::any::Any;
 use std::collections::{BTreeMap, HashMap};
-use std::pin::Pin;
 use std::sync::Mutex;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::{ops::Range, sync::Arc};
+use std::task::Poll;
+use std::{
+    ops::{Range, RangeInclusive},
+    sync::Arc,
+};
 
-use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt64Type;
+use arrow_array::{Array, BooleanArray, RecordBatch, UInt32Array};
+use arrow_schema::{Schema as ArrowSchema, SchemaRef};
 use datafusion::common::runtime::SpawnedTask;
 use datafusion::common::stats::Precision;
 use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::stream::{RecordBatchReceiverStream, RecordBatchStreamAdapter};
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
     execution_plan::{Boundedness, EmissionType},
@@ -31,7 +35,9 @@ use lance_core::datatypes::OnMissing;
 use lance_core::utils::deletion::DeletionVector;
 use lance_core::utils::futures::FinallyStreamExt;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
-use lance_core::{Error, Result, datatypes::Projection};
+use lance_core::{
+    Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_ID, ROW_ID_FIELD, Result, datatypes::Projection,
+};
 use lance_datafusion::planner::Planner;
 use lance_datafusion::utils::{
     ExecutionPlanMetricsSetExt, FRAGMENTS_SCANNED_METRIC, RANGES_SCANNED_METRIC,
@@ -41,7 +47,8 @@ use lance_file::reader::FileReaderOptions;
 use lance_index::scalar::expression::FilterPlan;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_select::{
-    IndexExprResult, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges, ranges_to_bitmap,
+    IndexExprResult, RowAddrMask, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges,
+    ranges_to_bitmap, result::IndexExprResultWireFormat,
 };
 use lance_table::format::Fragment;
 use lance_table::rowids::RowIdSequence;
@@ -60,6 +67,13 @@ use crate::dataset::scanner::{
 
 use super::utils::IoMetrics;
 
+fn public_blob_v2_binary_projection_schema(projection: &Projection) -> SchemaRef {
+    let schema = projection.to_schema();
+    let schema = crate::dataset::blob::public_blob_v2_binary_output_schema(&schema);
+    let schema: ArrowSchema = (&schema).into();
+    Arc::new(schema)
+}
+
 #[derive(Debug)]
 pub struct EvaluatedIndex {
     index_result: IndexExprResult,
@@ -86,6 +100,18 @@ impl EvaluatedIndex {
             applicable_fragments,
         })
     }
+
+    /// Block `rows` (stale overlay row addresses) from the index result so the index never
+    /// emits them. Their fragments stay in the covered set, so non-stale rows keep the index;
+    /// the blocked rows are re-evaluated against their current (overlay-merged) values on a
+    /// separate targeted take path built by the scanner.
+    fn without_rows(mut self, block_list: &RowAddrTreeMap) -> Self {
+        self.index_result.upper =
+            std::mem::take(&mut self.index_result.upper).also_block(block_list.clone());
+        self.index_result.lower =
+            std::mem::take(&mut self.index_result.lower).also_block(block_list.clone());
+        self
+    }
 }
 
 /// A fragment along with ranges of row offsets to read
@@ -328,6 +354,116 @@ struct FilteredReadStream {
     threading_mode: FilteredReadThreadingMode,
     /// Range to apply to the result stream if not already pushed down in planning phase
     scan_range_after_filter: Option<Range<u64>>,
+    /// Fragments planned non-empty, and their total planned rows; the output
+    /// side uses these to detect take-shaped plans (batch size resolves at
+    /// execute time, so the detection lives there too)
+    touched_fragments: usize,
+    planned_rows: u64,
+}
+
+/// Below this many fragments there are too few handoffs to be worth
+/// consolidating
+const CONSOLIDATE_MIN_FRAGMENTS: usize = 8;
+
+/// Above this per-fragment average, batches are big enough to amortize
+/// their handoff
+const CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT: u64 = 1024;
+
+/// Pump a take-shaped read on a spawned task, handing the consumer
+/// consolidated batches. Inline polling would otherwise execute the
+/// per-batch pipeline work on the consumer, which serializes concurrent
+/// small reads.
+fn consolidated_stream(
+    inner: SendableRecordBatchStream,
+    target: usize,
+) -> SendableRecordBatchStream {
+    let mut builder = RecordBatchReceiverStream::builder(inner.schema(), 4);
+    let tx = builder.tx();
+    builder.spawn(async move {
+        let mut stream = coalesce_batches(inner, target).boxed();
+        while let Some(item) = stream.next().await {
+            if tx.send(item).await.is_err() {
+                // Receiver dropped: the query was cancelled
+                break;
+            }
+        }
+        Ok(())
+    });
+    builder.build()
+}
+
+/// Merge batches up to `target` rows; batches already at the target pass
+/// through whole (never split). Order is preserved.
+pub fn coalesce_batches(
+    input: SendableRecordBatchStream,
+    target: usize,
+) -> impl Stream<Item = DataFusionResult<RecordBatch>> {
+    struct Coalescer {
+        input: SendableRecordBatchStream,
+        schema: SchemaRef,
+        target: usize,
+        buffered: Vec<RecordBatch>,
+        buffered_rows: usize,
+        exhausted: bool,
+    }
+
+    impl Coalescer {
+        fn ready_to_emit(&self) -> bool {
+            self.buffered_rows >= self.target || (self.exhausted && !self.buffered.is_empty())
+        }
+
+        fn buffer(&mut self, batch: RecordBatch) {
+            self.buffered_rows += batch.num_rows();
+            self.buffered.push(batch);
+        }
+
+        fn emit(&mut self) -> DataFusionResult<RecordBatch> {
+            self.buffered_rows = 0;
+            if self.buffered.len() > 1 {
+                let batch = arrow::compute::concat_batches(&self.schema, self.buffered.iter())?;
+                self.buffered.clear();
+                Ok(batch)
+            } else {
+                self.buffered.pop().ok_or_else(|| {
+                    DataFusionError::Internal(
+                        "coalesce_batches emitted with an empty buffer".to_string(),
+                    )
+                })
+            }
+        }
+    }
+
+    let schema = input.schema();
+    let coalescer = Coalescer {
+        input,
+        schema,
+        target,
+        buffered: Vec::new(),
+        buffered_rows: 0,
+        exhausted: false,
+    };
+    futures::stream::try_unfold(coalescer, |mut this| async move {
+        loop {
+            if this.ready_to_emit() {
+                return Ok(Some((this.emit()?, this)));
+            }
+            if this.exhausted {
+                return Ok(None);
+            }
+            match this.input.try_next().await? {
+                Some(batch) if batch.num_rows() >= this.target && !this.buffered.is_empty() => {
+                    // Emit the partial buffer on its own; the large batch
+                    // then passes through whole on the next iteration
+                    let out = this.emit()?;
+                    this.buffer(batch);
+                    return Ok(Some((out, this)));
+                }
+                Some(batch) if batch.num_rows() > 0 => this.buffer(batch),
+                Some(_) => {}
+                None => this.exhausted = true,
+            }
+        }
+    })
 }
 
 impl std::fmt::Debug for FilteredReadStream {
@@ -337,16 +473,22 @@ impl std::fmt::Debug for FilteredReadStream {
 }
 
 impl FilteredReadStream {
-    /// Create a new FilteredReadStream from a pre-computed internal plan
+    /// Create a new FilteredReadStream from a pre-computed internal plan.
+    /// Fragment handles are constructed I/O-free from the manifest
+    /// descriptors, only for the fragments the plan selects. A `None`
+    /// scheduler is created here; the row-stream path injects its per-query
+    /// shared one and a per-batch priority offset.
     #[instrument(name = "init_filtered_read_stream", skip_all)]
-    async fn try_new(
+    fn try_new(
         dataset: Arc<Dataset>,
         options: FilteredReadOptions,
-        metrics: &ExecutionPlanMetricsSet,
+        global_metrics: Arc<FilteredReadGlobalMetrics>,
         plan: FilteredReadInternalPlan,
-    ) -> DataFusionResult<Self> {
-        let global_metrics = Arc::new(FilteredReadGlobalMetrics::new(metrics));
-
+        scan_scheduler: Option<Arc<ScanScheduler>>,
+        priority_offset: Option<u32>,
+    ) -> Self {
+        let scan_scheduler =
+            scan_scheduler.unwrap_or_else(|| Self::make_scan_scheduler(&dataset, &options));
         let threading_mode = options.threading_mode;
 
         let io_parallelism = dataset.object_store.io_parallelism();
@@ -355,62 +497,36 @@ impl FilteredReadStream {
             .unwrap_or_else(|| (*DEFAULT_FRAGMENT_READAHEAD).unwrap_or(io_parallelism * 2))
             .max(1);
 
-        let fragments = options
+        let fragment_descriptors = options
             .fragments
             .clone()
             .unwrap_or_else(|| dataset.fragments().clone());
 
         log::debug!(
             "Filtered read on {} fragments with frag_readahead={} and io_parallelism={}",
-            fragments.len(),
+            fragment_descriptors.len(),
             fragment_readahead,
             io_parallelism
         );
 
-        // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is
-        // not general enough" false positives from rustc
-        let frag_futs = fragments
-            .iter()
-            .map(|frag| {
-                Result::Ok(Self::load_fragment(
-                    dataset.clone(),
-                    frag.clone(),
-                    options.with_deleted_rows,
-                ))
-            })
-            .collect::<Vec<_>>();
-        let loaded_fragments = futures::stream::iter(frag_futs)
-            // Cannot use unordered because we need to populate logical_offset based on user-provided order
-            .try_buffered(io_parallelism)
-            .try_collect::<Vec<_>>()
-            .await?;
-
-        let output_schema = Arc::new(options.projection.to_arrow_schema());
-
-        let obj_store = dataset.object_store.clone();
-        // Explicit options take precedence; otherwise fall back to the
-        // LANCE_DEFAULT_IO_BUFFER_SIZE env var if set; otherwise max_bandwidth.
-        let scheduler_config = if let Some(io_buffer_size_bytes) = options
-            .io_buffer_size_bytes
-            .or_else(get_default_io_buffer_size_override)
-        {
-            SchedulerConfig::new(io_buffer_size_bytes)
-        } else {
-            SchedulerConfig::max_bandwidth(obj_store.as_ref())
-        };
-        let scan_scheduler = ScanScheduler::new(obj_store, scheduler_config);
+        let output_schema = public_blob_v2_binary_projection_schema(&options.projection);
 
         // Get scan_range_after_filter from the plan
         let scan_range_after_filter = plan.scan_range_after_filter.clone();
 
         // Convert plan to scoped fragments for I/O
-        let scoped_fragments = Self::plan_to_scoped_fragments(
+        let mut scoped_fragments = Self::plan_to_scoped_fragments(
             &plan,
-            &loaded_fragments,
+            &fragment_descriptors,
             &dataset,
             &options,
             scan_scheduler.clone(),
         );
+        if let Some(priority_offset) = priority_offset.filter(|offset| *offset != 0) {
+            for scoped in &mut scoped_fragments {
+                scoped.priority = scoped.priority.saturating_add(priority_offset);
+            }
+        }
 
         let global_metrics_clone = global_metrics.clone();
 
@@ -420,8 +536,10 @@ impl FilteredReadStream {
                 move |scoped_fragment| {
                     let metrics = global_metrics_clone.clone();
                     let limit = scan_range_after_filter.as_ref().map(|r| r.end);
+                    let dataset = dataset.clone();
                     SpawnedTask::spawn(
-                        Self::read_fragment(scoped_fragment, metrics, limit).in_current_span(),
+                        Self::read_fragment(dataset, scoped_fragment, metrics, limit)
+                            .in_current_span(),
                     )
                     .map(|thread_result| thread_result.unwrap())
                 }
@@ -429,7 +547,23 @@ impl FilteredReadStream {
             .buffered(fragment_readahead);
         let task_stream = fragment_streams.try_flatten().boxed();
 
-        Ok(Self {
+        // A batch never spans fragments, so a plan touching many fragments
+        // with few rows each emits one tiny batch per fragment. Fragments
+        // planned empty produce no batch and don't count. Filtered scans
+        // stay dense here: their planned rows are a pre-refine upper bound.
+        let (touched_fragments, planned_rows) =
+            plan.rows
+                .values()
+                .fold((0usize, 0u64), |(fragments, rows), ranges| {
+                    let fragment_rows: u64 =
+                        ranges.iter().map(|range| range.end - range.start).sum();
+                    if fragment_rows > 0 {
+                        (fragments + 1, rows + fragment_rows)
+                    } else {
+                        (fragments, rows)
+                    }
+                });
+        Self {
             output_schema,
             task_stream: Arc::new(AsyncMutex::new(task_stream)),
             scan_scheduler,
@@ -437,7 +571,62 @@ impl FilteredReadStream {
             active_partitions_counter: Arc::new(AtomicUsize::new(0)),
             threading_mode,
             scan_range_after_filter,
-        })
+            touched_fragments,
+            planned_rows,
+        }
+    }
+
+    /// Drain the entire read into batches (used by the row-stream path,
+    /// which is the stream's only consumer and records metrics per batch)
+    async fn collect_all(&self, decode_parallelism: usize) -> Result<Vec<RecordBatch>> {
+        let mut task_stream = self.task_stream.lock().await;
+        (&mut *task_stream)
+            .try_buffered(decode_parallelism)
+            .try_collect()
+            .await
+    }
+
+    async fn load_all_fragments(
+        dataset: &Arc<Dataset>,
+        options: &FilteredReadOptions,
+    ) -> Result<Vec<LoadedFragment>> {
+        let io_parallelism = dataset.object_store.io_parallelism();
+        let fragments = options
+            .fragments
+            .clone()
+            .unwrap_or_else(|| dataset.fragments().clone());
+        // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is
+        // not general enough" false positives from rustc
+        let frag_futs = fragments
+            .iter()
+            .map(|frag| {
+                Result::Ok(Self::load_fragment(
+                    dataset.clone(),
+                    frag.clone(),
+                    options.with_deleted_rows,
+                ))
+            })
+            .collect::<Vec<_>>();
+        futures::stream::iter(frag_futs)
+            // Cannot use unordered because we need to populate logical_offset based on user-provided order
+            .try_buffered(io_parallelism)
+            .try_collect::<Vec<_>>()
+            .await
+    }
+
+    /// Create the I/O scheduler for a read (explicit option → env override →
+    /// max bandwidth)
+    fn make_scan_scheduler(dataset: &Dataset, options: &FilteredReadOptions) -> Arc<ScanScheduler> {
+        let obj_store = dataset.object_store.clone();
+        let scheduler_config = if let Some(io_buffer_size_bytes) = options
+            .io_buffer_size_bytes
+            .or_else(get_default_io_buffer_size_override)
+        {
+            SchedulerConfig::new(io_buffer_size_bytes)
+        } else {
+            SchedulerConfig::max_bandwidth(obj_store.as_ref())
+        };
+        ScanScheduler::new(obj_store, scheduler_config)
     }
 
     async fn load_fragment(
@@ -640,10 +829,13 @@ impl FilteredReadStream {
         }
     }
 
+    /// Handles are constructed here, I/O-free, only for the fragments the
+    /// plan selects; priority is the fragment's position in the candidate
+    /// list so a sparse plan keeps the original I/O order.
     fn plan_to_scoped_fragments(
         plan: &FilteredReadInternalPlan,
-        fragments: &[LoadedFragment],
-        dataset: &Dataset,
+        fragments: &[Fragment],
+        dataset: &Arc<Dataset>,
         options: &FilteredReadOptions,
         scan_scheduler: Arc<ScanScheduler>,
     ) -> Vec<ScopedFragmentRead> {
@@ -659,7 +851,7 @@ impl FilteredReadStream {
         let mut scoped_fragments = Vec::new();
 
         for (priority, fragment) in fragments.iter().enumerate() {
-            let fragment_id = fragment.fragment.id() as u32;
+            let fragment_id = fragment.id as u32;
 
             // Check if this fragment is in the plan
             if let Some(ranges) = plan.rows.get(&fragment_id) {
@@ -671,7 +863,7 @@ impl FilteredReadStream {
                 let filter = plan.filters.get(&fragment_id).map(|f| (**f).clone());
 
                 scoped_fragments.push(ScopedFragmentRead {
-                    fragment: fragment.fragment.clone(),
+                    fragment: Arc::new(FileFragment::new(dataset.clone(), fragment.clone())),
                     ranges: ranges.clone(),
                     projection: projection.clone(),
                     with_deleted_rows: options.with_deleted_rows,
@@ -1076,11 +1268,13 @@ impl FilteredReadStream {
     // Reads a single fragment into a stream of batch tasks
     #[instrument(name = "read_fragment", level = "debug", skip_all)]
     async fn read_fragment(
+        dataset: Arc<Dataset>,
         mut fragment_read_task: ScopedFragmentRead,
         global_metrics: Arc<FilteredReadGlobalMetrics>,
         fragment_soft_limit: Option<u64>,
     ) -> Result<BoxStream<'static, Result<ReadBatchFut>>> {
-        let output_schema = Arc::new(fragment_read_task.projection.to_arrow_schema());
+        let output_schema =
+            public_blob_v2_binary_projection_schema(fragment_read_task.projection.as_ref());
 
         if let Some(filter) = &fragment_read_task.filter {
             let filter_cols = Planner::column_names_in_expr(filter);
@@ -1095,10 +1289,22 @@ impl FilteredReadStream {
             }
         }
 
-        let read_schema = fragment_read_task.projection.to_bare_schema();
+        let output_read_schema = Arc::new(fragment_read_task.projection.to_schema());
+        let bare_read_schema = fragment_read_task.projection.to_bare_schema();
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(&bare_read_schema);
+        let read_schema = if materialize_blob_v2_binary {
+            crate::dataset::blob::blob_v2_descriptor_schema(&bare_read_schema)
+        } else {
+            bare_read_schema
+        };
+        let mut frag_read_config = fragment_read_task.frag_read_config();
+        if materialize_blob_v2_binary {
+            frag_read_config = frag_read_config.with_row_address(true);
+        }
         let mut fragment_reader = fragment_read_task
             .fragment
-            .open(&read_schema, fragment_read_task.frag_read_config())
+            .open(&read_schema, frag_read_config)
             .await?;
 
         if fragment_read_task.with_deleted_rows {
@@ -1112,8 +1318,9 @@ impl FilteredReadStream {
         let physical_filter = fragment_read_task
             .filter
             .map(|filter| {
-                let planner =
-                    Planner::new(Arc::new(fragment_read_task.projection.to_arrow_schema()));
+                let planner = Planner::new(public_blob_v2_binary_projection_schema(
+                    fragment_read_task.projection.as_ref(),
+                ));
                 planner.create_physical_expr(&filter)
             })
             .transpose()?;
@@ -1136,7 +1343,7 @@ impl FilteredReadStream {
                 let global_metrics = global_metrics.clone();
                 let fragment_counted = fragment_counted.clone();
                 let range_tracker = range_tracker.clone();
-                batch_fut
+                let batch_fut = batch_fut
                     .inspect_ok(move |batch| {
                         let num_rows = batch.num_rows();
                         global_metrics.rows_scanned.add(num_rows);
@@ -1151,7 +1358,23 @@ impl FilteredReadStream {
                             global_metrics.ranges_scanned.add(additional_ranges);
                         }
                     })
-                    .boxed()
+                    .boxed();
+                if materialize_blob_v2_binary {
+                    let dataset = dataset.clone();
+                    let output_read_schema = output_read_schema.clone();
+                    batch_fut
+                        .and_then(move |batch| async move {
+                            crate::dataset::blob::materialize_blob_v2_binary_batch(
+                                &dataset,
+                                output_read_schema.as_ref(),
+                                batch,
+                            )
+                            .await
+                        })
+                        .boxed()
+                } else {
+                    batch_fut
+                }
             })
             .zip(futures::stream::repeat((
                 physical_filter.clone(),
@@ -1159,12 +1382,11 @@ impl FilteredReadStream {
             )))
             .map(|(batch_fut, args)| Self::wrap_with_filter(batch_fut, args.0, args.1));
 
-        let result: Pin<Box<dyn Stream<Item = Result<ReadBatchFut>> + Send>> =
-            if let Some(limit) = fragment_soft_limit {
-                Box::pin(Self::apply_soft_limit(fragment_stream, limit))
-            } else {
-                Box::pin(fragment_stream)
-            };
+        let result = if let Some(limit) = fragment_soft_limit {
+            Self::apply_soft_limit(fragment_stream, limit).boxed()
+        } else {
+            fragment_stream.boxed()
+        };
         Ok(result)
     }
 
@@ -1270,7 +1492,7 @@ pub struct FilteredReadOptions {
     pub scan_range_before_filter: Option<Range<u64>>,
     /// The range of rows to read after applying the filter.
     pub scan_range_after_filter: Option<Range<u64>>,
-    /// Include deleted rows in the scan
+    /// Include deleted rows in the scan; they are returned with a null row id
     pub with_deleted_rows: bool,
     /// The maximum number of rows per batch
     pub batch_size: Option<u32>,
@@ -1295,6 +1517,12 @@ pub struct FilteredReadOptions {
     pub io_buffer_size_bytes: Option<u64>,
     /// If true, skip fragments that are not covered by the scalar index result.
     pub only_indexed_fragments: bool,
+    /// Row addresses whose index entries may be stale because an overlay committed after the
+    /// index was built touches an indexed field. They are blocked from the index result so the
+    /// index never emits them; the scanner re-evaluates just these rows against their current
+    /// (overlay-merged) values on a targeted take path. Their fragments stay in the covered set,
+    /// so non-stale rows keep the index. `None` on the common no-overlay fast path.
+    pub overlay_block: Option<RowAddrMask>,
 }
 
 impl FilteredReadOptions {
@@ -1324,12 +1552,20 @@ impl FilteredReadOptions {
             full_filter: None,
             io_buffer_size_bytes: None,
             only_indexed_fragments: false,
+            overlay_block: None,
             threading_mode: FilteredReadThreadingMode::OnePartitionMultipleThreads(
                 get_num_compute_intensive_cpus(),
             ),
         }
     }
 
+    /// Block the given stale overlay row addresses (see the `overlay_block` field) from the
+    /// scalar index result so the index never emits them.
+    pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self {
+        self.overlay_block = Some(block);
+        self
+    }
+
     /// Include deleted rows in the scan
     ///
     /// This is currently only supported if there is no scan_range specified
@@ -1482,6 +1718,19 @@ impl FilteredReadOptions {
         self.only_indexed_fragments = true;
         self
     }
+
+    /// Specify the threading mode to use for the scan.
+    ///
+    /// This controls how decode work is parallelized.  For the default single-partition
+    /// scan, the parameter of [`FilteredReadThreadingMode::OnePartitionMultipleThreads`]
+    /// bounds how many batch-decode tasks are buffered in flight (via `try_buffered`).
+    ///
+    /// The parallelism must be greater than 0.  A value of 0 is rejected by
+    /// [`FilteredReadExec::try_new`].
+    pub fn with_threading_mode(mut self, threading_mode: FilteredReadThreadingMode) -> Self {
+        self.threading_mode = threading_mode;
+        self
+    }
 }
 
 /// A plan node that reads a dataset, applying an optional filter and projection.
@@ -1504,7 +1753,7 @@ pub struct FilteredReadExec {
     options: FilteredReadOptions,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
-    index_input: Option<Arc<dyn ExecutionPlan>>,
+    input: RowSelector,
     // Precomputed internal plan
     plan: Arc<OnceCell<FilteredReadInternalPlan>>,
     // When execute is first called we will initialize the FilteredReadStream.  In order to support
@@ -1512,6 +1761,49 @@ pub struct FilteredReadExec {
     running_stream: Arc<AsyncMutex<Option<FilteredReadStream>>>,
 }
 
+/// Describes which rows a [`FilteredReadExec`] should read
+#[derive(Debug)]
+enum RowSelector {
+    /// Every live row of the dataset (no input plan)
+    AllRows,
+    /// A set of rows: one serialized [`IndexExprResult`] batch.  Output is in
+    /// storage order and deduplicated.
+    RowSet(Arc<dyn ExecutionPlan>),
+    /// A stream of rows: record batches with a `_rowid`/`_rowaddr` column
+    /// and other payload columns (just carried)
+    RowStream(Arc<RowStreamSource>),
+}
+
+impl RowSelector {
+    fn row_set_plan(&self) -> Option<&Arc<dyn ExecutionPlan>> {
+        match self {
+            Self::RowSet(plan) => Some(plan),
+            _ => None,
+        }
+    }
+
+    fn child(&self) -> Option<&Arc<dyn ExecutionPlan>> {
+        match self {
+            Self::AllRows => None,
+            Self::RowSet(plan) => Some(plan),
+            Self::RowStream(source) => Some(&source.plan),
+        }
+    }
+}
+
+/// State derived at construction for a row-stream source
+#[derive(Debug)]
+struct RowStreamSource {
+    plan: Arc<dyn ExecutionPlan>,
+    /// The stream column identifying rows: [`ROW_ID`] or [`ROW_ADDR`]
+    key_column: &'static str,
+    /// Options for the internal fragment read; carries the projection that
+    /// reflects the actual columns to read (plus the alignment key column)
+    read_options: FilteredReadOptions,
+    /// The schema for newly read columns
+    new_fields_schema: SchemaRef,
+}
+
 /// Public plan for distributed execution - uses bitmap for flexibility
 #[derive(Clone)]
 pub struct FilteredReadPlan {
@@ -1558,11 +1850,170 @@ impl FilteredReadInternalPlan {
 }
 
 impl FilteredReadExec {
+    /// Create a new filtered read
     pub fn try_new(
+        dataset: Arc<Dataset>,
+        options: FilteredReadOptions,
+        input: Option<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Self> {
+        match input {
+            Some(input) if Self::is_index_query_schema(input.schema().as_ref()) => {
+                Self::try_new_scan(dataset, options, Some(input))
+            }
+            Some(input) => Self::try_new_row_stream(dataset, options, input),
+            None => Self::try_new_scan(dataset, options, None),
+        }
+    }
+
+    /// Whether `schema` is one of the serialized [`IndexExprResult`] wire
+    /// layouts (see [`IndexExprResultWireFormat`])
+    fn is_index_query_schema(schema: &arrow_schema::Schema) -> bool {
+        [
+            IndexExprResultWireFormat::TwoMask,
+            IndexExprResultWireFormat::ThreeVariant,
+        ]
+        .iter()
+        .any(|format| schema.fields() == format.schema().fields())
+    }
+
+    /// The input columns that carry through to the output: identity columns
+    /// appear iff their flag is requested, ordinary columns always carry
+    fn carried_schema(input_schema: &arrow_schema::Schema, projection: &Projection) -> SchemaRef {
+        Arc::new(arrow_schema::Schema::new(
+            input_schema
+                .fields()
+                .iter()
+                .filter(|f| {
+                    (f.name() != ROW_ID || projection.with_row_id)
+                        && (f.name() != ROW_ADDR || projection.with_row_addr)
+                })
+                .cloned()
+                .collect::<Vec<_>>(),
+        ))
+    }
+
+    /// Construct a read over a row-stream source
+    fn try_new_row_stream(
+        dataset: Arc<Dataset>,
+        options: FilteredReadOptions,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<Self> {
+        if dataset.is_legacy_storage() {
+            return Err(Error::not_supported_source(
+                "taking rows through FilteredReadExec requires the v2 storage format"
+                    .to_string()
+                    .into(),
+            ));
+        }
+        if options.refine_filter.is_some() || options.full_filter.is_some() {
+            return Err(Error::invalid_input_source(
+                "filters are not supported when taking rows from an input plan".into(),
+            ));
+        }
+        // A limit is safer to apply upstream, on the cheap keyed rows
+        if options.scan_range_before_filter.is_some() || options.scan_range_after_filter.is_some() {
+            return Err(Error::invalid_input_source(
+                "scan ranges are not supported when taking rows from an input plan".into(),
+            ));
+        }
+        // Row-stream reads do not support deleted rows yet; deleted rows are
+        // excluded from the output by default
+        if options.with_deleted_rows || options.only_indexed_fragments {
+            return Err(Error::invalid_input_source(
+                "with_deleted_rows / only_indexed_fragments are not supported when taking rows from an input plan".into(),
+            ));
+        }
+        let input_schema = input.schema();
+        let key_column = if input_schema.column_with_name(ROW_ID).is_some() {
+            ROW_ID
+        } else if input_schema.column_with_name(ROW_ADDR).is_some() {
+            ROW_ADDR
+        } else {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "a row-stream input plan must have a column named '{}' or '{}'",
+                    ROW_ADDR, ROW_ID
+                )
+                .into(),
+            ));
+        };
+
+        let fields_to_read = options
+            .projection
+            .clone()
+            .subtract_arrow_schema(input_schema.as_ref(), OnMissing::Ignore)?;
+        let synthesize_row_id = fields_to_read.with_row_id;
+        let synthesize_row_addr = fields_to_read.with_row_addr;
+        if !fields_to_read.has_data_fields() && !synthesize_row_id && !synthesize_row_addr {
+            return Err(Error::invalid_input_source(
+                "the input plan already contains every projected field; there is nothing to read"
+                    .into(),
+            ));
+        }
+
+        let carried_schema = Self::carried_schema(input_schema.as_ref(), &options.projection);
+
+        // Output = carried columns ⊕ fetched fields ⊕ synthesized identity
+        let output_schema = Arc::new(arrow_schema::Schema::from(
+            &super::TakeExec::calculate_output_schema(
+                dataset.schema(),
+                carried_schema.as_ref(),
+                &fields_to_read,
+            ),
+        ));
+
+        // Partitioning and emission behavior follow the input
+        let properties = Arc::new(
+            input
+                .properties()
+                .as_ref()
+                .clone()
+                .with_eq_properties(EquivalenceProperties::new(output_schema)),
+        );
+
+        let bare_schema = arrow_schema::Schema::from(&fields_to_read.to_bare_schema());
+        let mut new_fields = bare_schema.fields().iter().cloned().collect::<Vec<_>>();
+        if synthesize_row_id {
+            new_fields.push(Arc::new(ROW_ID_FIELD.clone()));
+        }
+        if synthesize_row_addr {
+            new_fields.push(Arc::new(ROW_ADDR_FIELD.clone()));
+        }
+        let new_fields_schema = Arc::new(arrow_schema::Schema::new(new_fields));
+
+        // fields_to_read keeps the synthesis flags; add the key column on top
+        let mut read_options = options.clone();
+        read_options.projection = if key_column == ROW_ID {
+            fields_to_read.with_row_id()
+        } else {
+            fields_to_read.with_row_addr()
+        };
+
+        Ok(Self {
+            dataset,
+            options,
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+            input: RowSelector::RowStream(Arc::new(RowStreamSource {
+                plan: input,
+                key_column,
+                read_options,
+                new_fields_schema,
+            })),
+            plan: Arc::new(OnceCell::new()),
+            running_stream: Arc::new(AsyncMutex::new(None)),
+        })
+    }
+
+    fn try_new_scan(
         dataset: Arc<Dataset>,
         mut options: FilteredReadOptions,
         index_input: Option<Arc<dyn ExecutionPlan>>,
     ) -> Result<Self> {
+        let input = match index_input {
+            Some(plan) => RowSelector::RowSet(plan),
+            None => RowSelector::AllRows,
+        };
         if options.with_deleted_rows {
             // Ensure we have the row id column if with_deleted_rows is set
             options.projection = options.projection.with_row_id();
@@ -1573,11 +2024,28 @@ impl FilteredReadExec {
                 .into()));
         }
 
+        // A parallelism of 0 would cause `try_buffered(0)` to hang forever instead of erroring
+        match options.threading_mode {
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(0) => {
+                return Err(Error::invalid_input_source(
+                    "FilteredReadThreadingMode::OnePartitionMultipleThreads must be greater than 0, got 0"
+                        .into(),
+                ));
+            }
+            FilteredReadThreadingMode::MultiplePartitions(0) => {
+                return Err(Error::invalid_input_source(
+                    "FilteredReadThreadingMode::MultiplePartitions must be greater than 0, got 0"
+                        .into(),
+                ));
+            }
+            _ => {}
+        }
+
         if options.scan_range_after_filter.is_some() {
             // Validate that there's a filter when using scan_range_after_filter
             if options.full_filter.is_none()
                 && options.refine_filter.is_none()
-                && index_input.is_none()
+                && input.row_set_plan().is_none()
             {
                 return Err(Error::invalid_input_source("scan_range_after_filter requires a filter to be applied. Use scan_range_before_filter for unfiltered scans."
                     .into()));
@@ -1595,7 +2063,7 @@ impl FilteredReadExec {
                 ));
             }
         }
-        let output_schema = Arc::new(options.projection.to_arrow_schema());
+        let output_schema = public_blob_v2_binary_projection_schema(&options.projection);
         let num_partitions = match options.threading_mode {
             FilteredReadThreadingMode::OnePartitionMultipleThreads(_) => 1,
             FilteredReadThreadingMode::MultiplePartitions(n) => n,
@@ -1616,7 +2084,7 @@ impl FilteredReadExec {
             properties,
             running_stream: Arc::new(AsyncMutex::new(None)),
             metrics,
-            index_input,
+            input,
             plan: Arc::new(OnceCell::new()),
         })
     }
@@ -1675,9 +2143,15 @@ impl FilteredReadExec {
                     let index_search_result = index_search.next().await.ok_or_else(|| {
                         Error::internal("Index search did not yield any results".to_string())
                     })??;
-                    evaluated_index = Some(Arc::new(EvaluatedIndex::try_from_arrow(
-                        &index_search_result,
-                    )?));
+                    let mut idx = EvaluatedIndex::try_from_arrow(&index_search_result)?;
+                    // `overlay_block` is always constructed as a block list (see
+                    // `Scanner::stale_rows_block_mask`), so `block_list()` is always `Some`.
+                    if let Some(block_list) =
+                        options.overlay_block.as_ref().and_then(|b| b.block_list())
+                    {
+                        idx = idx.without_rows(block_list);
+                    }
+                    evaluated_index = Some(Arc::new(idx));
                 }
 
                 // Load fragments to compute the plan
@@ -1703,7 +2177,9 @@ impl FilteredReadExec {
                     .try_collect::<Vec<_>>()
                     .await?;
 
-                // Plan the scan
+                // Plan the scan; the metadata loaded here drops when planning
+                // finishes — stream construction rebuilds I/O-free handles
+                // from the manifest descriptors
                 Ok(FilteredReadStream::plan_scan(
                     &loaded_fragments,
                     &evaluated_index,
@@ -1715,11 +2191,18 @@ impl FilteredReadExec {
 
     /// Get the existing plan or create it if it doesn't exist
     pub async fn get_or_create_plan(&self, ctx: Arc<TaskContext>) -> Result<FilteredReadPlan> {
+        if self.row_stream_input().is_some() {
+            return Err(Error::not_supported_source(
+                "a FilteredReadExec with a row-stream source does not have a precomputable plan"
+                    .to_string()
+                    .into(),
+            ));
+        }
         let internal_plan = Self::get_or_create_plan_impl(
             &self.plan,
             self.dataset.clone(),
             &self.options,
-            self.index_input.as_ref(),
+            self.input.row_set_plan(),
             0,
             ctx,
         )
@@ -1746,12 +2229,13 @@ impl FilteredReadExec {
                 n.min(target_partitions).max(1),
             );
         }
+        let batch_size_rows = options.batch_size;
         let batch_size_bytes = options
             .file_reader_options
             .as_ref()
             .and_then(|o| o.batch_size_bytes);
         let metrics = self.metrics.clone();
-        let index_input = self.index_input.clone();
+        let index_input = self.input.row_set_plan().cloned();
         let plan_cell = self.plan.clone();
 
         let stream = futures::stream::once(async move {
@@ -1769,16 +2253,42 @@ impl FilteredReadExec {
                 )
                 .await
                 .map_err(|e| DataFusionError::External(e.into()))?;
-                let new_running_stream =
-                    FilteredReadStream::try_new(dataset, options, &metrics, plan.clone())
-                        .await
-                        .map_err(|e| DataFusionError::External(e.into()))?;
+                let new_running_stream = FilteredReadStream::try_new(
+                    dataset,
+                    options,
+                    Arc::new(FilteredReadGlobalMetrics::new(&metrics)),
+                    plan.clone(),
+                    None,
+                    None,
+                );
                 let first_stream = new_running_stream.get_stream(&metrics, partition);
                 *running_stream = Some(new_running_stream);
                 first_stream
             };
-            let stream: SendableRecordBatchStream = match batch_size_bytes {
-                Some(target) => {
+            // Only masked reads consolidate; plain scans keep their batch
+            // boundaries, and the byte-based rechunk merges on its own
+            let consolidate = if index_input.is_some() && batch_size_bytes.is_none() {
+                running_stream.as_ref().and_then(|running| {
+                    // Explicit option → lance env default → session batch size
+                    let batch_target_rows = batch_size_rows
+                        .map(|batch_size| batch_size as usize)
+                        .or_else(get_default_batch_size)
+                        .unwrap_or_else(|| context.session_config().batch_size());
+                    let is_sparse_plan = batch_target_rows > 0
+                        && running.touched_fragments >= CONSOLIDATE_MIN_FRAGMENTS
+                        && running.planned_rows
+                            < running.touched_fragments as u64
+                                * CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT;
+                    is_sparse_plan.then_some(batch_target_rows)
+                })
+            } else {
+                None
+            };
+            drop(running_stream);
+
+            let stream = match (consolidate, batch_size_bytes) {
+                (Some(target), _) => consolidated_stream(inner, target),
+                (None, Some(bytes)) => {
                     let schema = inner.schema();
                     Box::pin(RecordBatchStreamAdapter::new(
                         schema.clone(),
@@ -1786,11 +2296,11 @@ impl FilteredReadExec {
                             inner,
                             schema,
                             0,
-                            target as usize,
+                            bytes as usize,
                         ),
                     ))
                 }
-                None => inner,
+                (None, None) => inner,
             };
             DataFusionResult::<SendableRecordBatchStream>::Ok(stream)
         })
@@ -1808,17 +2318,476 @@ impl FilteredReadExec {
     }
 
     pub fn index_input(&self) -> Option<&Arc<dyn ExecutionPlan>> {
-        self.index_input.as_ref()
+        self.input.row_set_plan()
+    }
+
+    pub fn row_stream_input(&self) -> Option<&Arc<dyn ExecutionPlan>> {
+        match &self.input {
+            RowSelector::RowStream(source) => Some(&source.plan),
+            _ => None,
+        }
     }
 
     /// Return the pre-computed plan if one exists, without triggering initialization.
     pub fn plan(&self) -> Option<FilteredReadPlan> {
         self.plan.get().map(|p| p.to_external_plan())
     }
+
+    fn execute_row_stream(
+        &self,
+        source: &Arc<RowStreamSource>,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> DataFusionResult<SendableRecordBatchStream> {
+        let input_stream = source.plan.execute(partition, context)?;
+        let dataset = self.dataset.clone();
+        let source = source.clone();
+        let carried_schema =
+            Self::carried_schema(source.plan.schema().as_ref(), &self.options.projection);
+        let output_schema = self.schema();
+        let metrics = self.metrics.clone();
+
+        let lazy_stream = futures::stream::once(async move {
+            let row_stream_read = Arc::new(RowStreamRead::new(
+                dataset,
+                source,
+                carried_schema,
+                output_schema,
+                &metrics,
+                partition,
+            ));
+            row_stream_read.apply(input_stream)
+        })
+        .flatten();
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            self.schema(),
+            lazy_stream,
+        )))
+    }
+}
+
+/// How many batches run concurrently.  Each batch's read already carries
+/// the full fragment-readahead and decode parallelism, so a shallow pipeline
+/// keeps the I/O pipe full; running every batch at once only multiplies that
+/// into lock contention
+const ROW_STREAM_CONCURRENT_BATCHES: usize = 4;
+
+/// Fragment metadata, loaded on the first batch and reused afterwards
+struct StreamFragments {
+    /// All dataset (or scoped) fragments, in dataset order
+    fragments: Vec<LoadedFragment>,
+    /// Fragment id → position in `fragments`
+    positions: HashMap<u32, usize>,
+    /// Each fragment's row-id span, for skipping fragments a batch cannot
+    /// touch (None = empty fragment)
+    id_spans: Vec<Option<RangeInclusive<u64>>>,
+}
+
+impl StreamFragments {
+    fn get(&self, fragment_id: u32) -> Option<&LoadedFragment> {
+        self.positions
+            .get(&fragment_id)
+            .map(|position| &self.fragments[*position])
+    }
+}
+
+/// Executes a [`FilteredReadExec`] over a row-stream source
+struct RowStreamRead {
+    dataset: Arc<Dataset>,
+    source: Arc<RowStreamSource>,
+    /// The input columns that carry through to the output
+    carried_schema: SchemaRef,
+    output_schema: SchemaRef,
+    scan_scheduler: Arc<ScanScheduler>,
+    loaded_fragments: OnceCell<StreamFragments>,
+    global_metrics: Arc<FilteredReadGlobalMetrics>,
+    baseline_metrics: BaselineMetrics,
+}
+
+impl RowStreamRead {
+    fn new(
+        dataset: Arc<Dataset>,
+        source: Arc<RowStreamSource>,
+        carried_schema: SchemaRef,
+        output_schema: SchemaRef,
+        metrics: &ExecutionPlanMetricsSet,
+        partition: usize,
+    ) -> Self {
+        let scan_scheduler =
+            FilteredReadStream::make_scan_scheduler(&dataset, &source.read_options);
+        Self {
+            dataset,
+            source,
+            carried_schema,
+            output_schema,
+            scan_scheduler,
+            loaded_fragments: OnceCell::new(),
+            global_metrics: Arc::new(FilteredReadGlobalMetrics::new(metrics)),
+            baseline_metrics: BaselineMetrics::new(metrics, partition),
+        }
+    }
+
+    async fn load_fragments(&self) -> Result<&StreamFragments> {
+        self.loaded_fragments
+            .get_or_try_init(|| async {
+                let fragments = FilteredReadStream::load_all_fragments(
+                    &self.dataset,
+                    &self.source.read_options,
+                )
+                .await?;
+                let positions = fragments
+                    .iter()
+                    .enumerate()
+                    .map(|(position, fragment)| (fragment.fragment.id() as u32, position))
+                    .collect();
+                let id_spans = fragments
+                    .iter()
+                    .map(|fragment| fragment.row_id_sequence.row_id_range())
+                    .collect();
+                Ok(StreamFragments {
+                    fragments,
+                    positions,
+                    id_spans,
+                })
+            })
+            .await
+    }
+
+    /// Build a batch's read ranges directly from physical row addresses
+    fn plan_batch_from_addresses(
+        addrs: &RowAddrTreeMap,
+        fragments: &StreamFragments,
+    ) -> FilteredReadInternalPlan {
+        let mut rows: BTreeMap<u32, Vec<Range<u64>>> = BTreeMap::new();
+        for (fragment_id, requested) in addrs.iter() {
+            // Unknown fragments (e.g. fully deleted) drop like stale keys
+            let Some(fragment) = fragments.get(*fragment_id) else {
+                continue;
+            };
+            let requested = match requested {
+                RowAddrSelection::Full => vec![0..fragment.num_physical_rows],
+                RowAddrSelection::Partial(bitmap) => bitmap_to_ranges(bitmap),
+            };
+            let valid = FilteredReadStream::full_frag_range(
+                fragment.num_physical_rows,
+                &fragment.deletion_vector,
+            );
+            let matched = FilteredReadStream::intersect_ranges(&valid, &requested);
+            if !matched.is_empty() {
+                rows.insert(*fragment_id, matched);
+            }
+        }
+        FilteredReadInternalPlan {
+            rows,
+            filters: HashMap::new(),
+            scan_range_after_filter: None,
+        }
+    }
+
+    /// Build a batch's read ranges by resolving stable row ids through the
+    /// fragments' row-id sequences
+    fn plan_batch_from_row_ids(
+        ids: RowAddrTreeMap,
+        keys: &arrow_array::PrimitiveArray<UInt64Type>,
+        fragments: &StreamFragments,
+    ) -> FilteredReadInternalPlan {
+        let mut rows: BTreeMap<u32, Vec<Range<u64>>> = BTreeMap::new();
+        let (Some(min_key), Some(max_key)) = (arrow::compute::min(keys), arrow::compute::max(keys))
+        else {
+            // Every key is null
+            return FilteredReadInternalPlan {
+                rows,
+                filters: HashMap::new(),
+                scan_range_after_filter: None,
+            };
+        };
+        let requested = RowAddrMask::from_allowed(ids);
+        for (fragment, id_span) in fragments.fragments.iter().zip(&fragments.id_spans) {
+            // Only fragments whose id span overlaps the batch's key range
+            // can hold requested rows
+            let Some(id_span) = id_span else { continue };
+            if *id_span.end() < min_key || *id_span.start() > max_key {
+                continue;
+            }
+            let offsets = fragment.row_id_sequence.mask_to_offset_ranges(&requested);
+            if offsets.is_empty() {
+                continue;
+            }
+            let valid = FilteredReadStream::full_frag_range(
+                fragment.num_physical_rows,
+                &fragment.deletion_vector,
+            );
+            let matched = FilteredReadStream::intersect_ranges(&valid, &offsets);
+            if !matched.is_empty() {
+                rows.insert(fragment.fragment.id() as u32, matched);
+            }
+        }
+        FilteredReadInternalPlan {
+            rows,
+            filters: HashMap::new(),
+            scan_range_after_filter: None,
+        }
+    }
+
+    fn key_array<'a>(
+        &self,
+        batch: &'a RecordBatch,
+        producer: &str,
+    ) -> DataFusionResult<&'a arrow_array::PrimitiveArray<UInt64Type>> {
+        let keys = batch
+            .column_by_name(self.source.key_column)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "the row-stream {} is missing the '{}' column",
+                    producer, self.source.key_column
+                ))
+            })?;
+        keys.as_primitive_opt::<UInt64Type>().ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "expected the row-stream column '{}' to be UInt64 but it was {}",
+                self.source.key_column,
+                keys.data_type()
+            ))
+        })
+    }
+
+    async fn plan_batch(
+        &self,
+        keys: &arrow_array::PrimitiveArray<UInt64Type>,
+    ) -> DataFusionResult<FilteredReadInternalPlan> {
+        let compute_timer = self.baseline_metrics.elapsed_compute().timer();
+        // Null keys are excluded; attach_columns drops their rows
+        let batch_keys = if keys.null_count() == 0 {
+            RowAddrTreeMap::from_iter(keys.values().iter().copied())
+        } else {
+            RowAddrTreeMap::from_iter(keys.iter().flatten())
+        };
+        drop(compute_timer);
+
+        let fragments = self.load_fragments().await?;
+        // Row ids equal row addresses when the dataset does not use stable
+        // row ids, so either key resolves directly by position
+        if self.source.key_column == ROW_ADDR || !self.dataset.manifest.uses_stable_row_ids() {
+            Ok(Self::plan_batch_from_addresses(&batch_keys, fragments))
+        } else {
+            Ok(Self::plan_batch_from_row_ids(batch_keys, keys, fragments))
+        }
+    }
+
+    /// Read the batch's planned ranges through the same executor as a scan,
+    /// returning the rows in storage order, deduplicated, with the key
+    /// column included
+    async fn read_batch(
+        &self,
+        internal_plan: FilteredReadInternalPlan,
+        batch_index: u32,
+    ) -> DataFusionResult<RecordBatch> {
+        let fragment_count = self.load_fragments().await?.fragments.len();
+        // I/O priority: earlier batches strictly first (output emits in batch
+        // order), fragments keep dataset order within a batch
+        let priority_offset = batch_index.saturating_mul(fragment_count as u32);
+        let read = FilteredReadStream::try_new(
+            self.dataset.clone(),
+            self.source.read_options.clone(),
+            self.global_metrics.clone(),
+            internal_plan,
+            Some(self.scan_scheduler.clone()),
+            Some(priority_offset),
+        );
+        let decode_parallelism = match self.source.read_options.threading_mode {
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(n) => n,
+            FilteredReadThreadingMode::MultiplePartitions(n) => n,
+        };
+        let read_batches = read.collect_all(decode_parallelism.max(1)).await?;
+        Ok(arrow::compute::concat_batches(
+            &read.output_schema,
+            read_batches.iter(),
+        )?)
+    }
+
+    /// Align the read rows back to the batch's row order and merge the
+    /// fetched columns on
+    fn attach_columns(
+        &self,
+        batch: RecordBatch,
+        read_data: RecordBatch,
+    ) -> DataFusionResult<RecordBatch> {
+        let _compute_timer = self.baseline_metrics.elapsed_compute().timer();
+        let keys = self.key_array(&batch, "input")?;
+        let read_keys = self.key_array(&read_data, "read")?;
+        attach_read_columns(
+            &batch,
+            keys,
+            &read_data,
+            read_keys,
+            self.carried_schema.as_ref(),
+            self.source.new_fields_schema.as_ref(),
+            &self.output_schema,
+        )
+    }
+
+    async fn execute_batch(
+        self: Arc<Self>,
+        batch: RecordBatch,
+        batch_index: u32,
+    ) -> DataFusionResult<RecordBatch> {
+        if batch.num_rows() == 0 {
+            return Ok(RecordBatch::new_empty(self.output_schema.clone()));
+        }
+        let internal_plan = self.plan_batch(self.key_array(&batch, "input")?).await?;
+        let read_data = self.read_batch(internal_plan, batch_index).await?;
+        self.attach_columns(batch, read_data)
+    }
+
+    fn apply(
+        self: Arc<Self>,
+        input: SendableRecordBatchStream,
+    ) -> impl Stream<Item = DataFusionResult<RecordBatch>> {
+        let batch_target_rows = self
+            .source
+            .read_options
+            .batch_size
+            .map(|batch_size| batch_size as usize)
+            .unwrap_or_else(|| get_default_batch_size().unwrap_or(BATCH_SIZE_FALLBACK));
+        let on_result = self.clone();
+        let on_done = self.clone();
+        coalesce_batches(input, batch_target_rows)
+            .enumerate()
+            .map(move |(batch_index, batch)| {
+                let batch = batch?;
+                let this = self.clone();
+                DataFusionResult::Ok(
+                    // SpawnedTask aborts on drop: cancelling the query
+                    // cancels in-flight batches
+                    SpawnedTask::spawn(
+                        this.execute_batch(batch, batch_index as u32)
+                            .in_current_span(),
+                    )
+                    .map(|res| match res {
+                        Ok(result) => result,
+                        Err(join_error) => Err(DataFusionError::External(Box::new(join_error))),
+                    }),
+                )
+            })
+            .boxed()
+            .try_buffered(ROW_STREAM_CONCURRENT_BATCHES)
+            .map(move |result| {
+                on_result
+                    .global_metrics
+                    .io_metrics
+                    .record(&on_result.scan_scheduler);
+                match on_result
+                    .baseline_metrics
+                    .record_poll(Poll::Ready(Some(result)))
+                {
+                    Poll::Ready(Some(result)) => result,
+                    _ => unreachable!("record_poll returned a different poll state"),
+                }
+            })
+            .finally(move || {
+                on_done.baseline_metrics.done();
+                on_done
+                    .global_metrics
+                    .io_metrics
+                    .record(&on_done.scan_scheduler);
+            })
+    }
+}
+
+/// Align `read_data` rows back to a keyed batch's row order and merge the
+/// fetched columns on.
+///
+/// `keys` is the row-id-space key of each `batch` row (usually the batch's
+/// own key column; a caller that keys by address passes the resolved ids
+/// instead) and `read_keys` the key of each `read_data` row, which must be
+/// unique. Input rows whose key has no read row — null or stale keys — are
+/// DROPPED; duplicate input keys re-expand through the gather. Output
+/// columns follow `output_schema`: `carried_schema` names come from `batch`,
+/// `new_fields_schema` names from `read_data`.
+pub fn attach_read_columns(
+    batch: &RecordBatch,
+    keys: &arrow_array::PrimitiveArray<UInt64Type>,
+    read_data: &RecordBatch,
+    read_keys: &arrow_array::PrimitiveArray<UInt64Type>,
+    carried_schema: &arrow_schema::Schema,
+    new_fields_schema: &arrow_schema::Schema,
+    output_schema: &SchemaRef,
+) -> DataFusionResult<RecordBatch> {
+    // Fast path: one read row per input row with an identical key sequence —
+    // already aligned, skip the hash map and the permutation
+    if keys.null_count() == 0
+        && read_data.num_rows() == batch.num_rows()
+        && read_keys.values() == keys.values()
+    {
+        let new_data = read_data.project_by_schema(new_fields_schema)?;
+        let carried = batch.project_by_schema(carried_schema)?;
+        return Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?);
+    }
+
+    let key_to_index: HashMap<u64, u32> = read_keys
+        .values()
+        .iter()
+        .enumerate()
+        .map(|(index, key)| (*key, index as u32))
+        .collect();
+
+    // Sizes differ only when some input keys have no live row (null or
+    // stale keys): drop those input rows first
+    let (batch, keys) = if read_data.num_rows() != batch.num_rows() {
+        let matched: BooleanArray = keys
+            .iter()
+            .map(|key| key.map(|key| key_to_index.contains_key(&key)))
+            .collect();
+        let keys = arrow::compute::filter(keys, &matched)?
+            .as_primitive::<UInt64Type>()
+            .clone();
+        (arrow::compute::filter_record_batch(batch, &matched)?, keys)
+    } else {
+        (batch.clone(), keys.clone())
+    };
+    if batch.num_rows() == 0 {
+        return Ok(RecordBatch::new_empty(output_schema.clone()));
+    }
+
+    // Gather the read rows into input order — every remaining key hits
+    let indices = UInt32Array::from_iter_values(keys.values().iter().map(|key| key_to_index[key]));
+    let new_data = arrow_select::take::take_record_batch(read_data, &indices)?;
+    let new_data = new_data.project_by_schema(new_fields_schema)?;
+    let carried = batch.project_by_schema(carried_schema)?;
+    Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?)
 }
 
 impl DisplayAs for FilteredReadExec {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if let RowSelector::RowStream(source) = &self.input {
+            let columns = source
+                .new_fields_schema
+                .fields
+                .iter()
+                .map(|f| f.name().as_str())
+                .collect::<Vec<_>>()
+                .join(", ");
+            return match t {
+                DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                    write!(
+                        f,
+                        "LanceRead: uri={}, projection=[{}], source=stream({})",
+                        self.dataset.data_dir(),
+                        columns,
+                        source.key_column,
+                    )
+                }
+                DisplayFormatType::TreeRender => {
+                    write!(
+                        f,
+                        "LanceRead\nuri={}\nprojection=[{}]\nsource=stream({})",
+                        self.dataset.data_dir(),
+                        columns,
+                        source.key_column,
+                    )
+                }
+            };
+        }
         let columns = self
             .options
             .projection
@@ -1892,22 +2861,24 @@ impl ExecutionPlan for FilteredReadExec {
         "FilteredReadExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
 
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        if let Some(index_input) = &self.index_input {
-            vec![index_input]
+        if let Some(child) = self.input.child() {
+            vec![child]
         } else {
             vec![]
         }
     }
 
+    fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+        // Partitioning a row-stream read would create multiple I/O schedulers
+        // (RAM heavy); the other selectors have no row input
+        vec![false; self.children().len()]
+    }
+
     fn metrics(&self) -> Option<MetricsSet> {
         Some(self.metrics.clone_inner())
     }
@@ -1915,7 +2886,14 @@ impl ExecutionPlan for FilteredReadExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion::error::Result<Statistics> {
+    ) -> datafusion::error::Result<Arc<Statistics>> {
+        if let RowSelector::RowStream(source) = &self.input {
+            // At most one output row per input row
+            return Ok(Arc::new(Statistics {
+                num_rows: source.plan.partition_statistics(partition)?.num_rows,
+                ..Statistics::new_unknown(self.schema().as_ref())
+            }));
+        }
         let fragments = self
             .options
             .fragments
@@ -1952,10 +2930,10 @@ impl ExecutionPlan for FilteredReadExec {
                 total_rows
             };
 
-            return Ok(Statistics {
+            return Ok(Arc::new(Statistics {
                 num_rows: Precision::Exact(total_rows as usize),
                 ..datafusion::physical_plan::Statistics::new_unknown(self.schema().as_ref())
-            });
+            }));
         };
 
         // We could evaluate the indexed filter here but this is still during the planning
@@ -1973,7 +2951,7 @@ impl ExecutionPlan for FilteredReadExec {
             .clone()
             .union_columns(filter_columns, OnMissing::Error)?;
 
-        let read_schema = Arc::new(read_projection.to_arrow_schema());
+        let read_schema = public_blob_v2_binary_projection_schema(&read_projection);
 
         let planner = Arc::new(Planner::new(read_schema.clone()));
         let physical_filter = planner.create_physical_expr(filter)?;
@@ -1990,7 +2968,7 @@ impl ExecutionPlan for FilteredReadExec {
             None,
         )?);
         let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?;
-        let mut df_stats = df_filter_exec.partition_statistics(partition)?;
+        let mut df_stats = Arc::unwrap_or_clone(df_filter_exec.partition_statistics(partition)?);
 
         // If we have an after-filter range, we should apply it to the stats (the before-filter range
         // is applied in the mock input)
@@ -2026,7 +3004,7 @@ impl ExecutionPlan for FilteredReadExec {
             }
         });
 
-        Ok(df_stats)
+        Ok(Arc::new(df_stats))
     }
 
     fn with_new_children(
@@ -2038,18 +3016,12 @@ impl ExecutionPlan for FilteredReadExec {
                 Error::internal("A FilteredReadExec cannot have two children".to_string()).into(),
             ))
         } else {
-            let index_input = children.into_iter().next();
-            Ok(Arc::new(Self {
-                dataset: self.dataset.clone(),
-                options: self.options.clone(),
-                properties: self.properties.clone(),
-                metrics: self.metrics.clone(),
-                // Seems unlikely this would already be initialized but clear it
-                // out just in case
-                running_stream: Arc::new(AsyncMutex::new(None)),
-                index_input,
-                plan: Arc::new(OnceCell::new()),
-            }))
+            // Rebuild via try_new so the selector and derived state are
+            // re-derived from the new child's schema
+            let child = children.into_iter().next();
+            let rebuilt = Self::try_new(self.dataset.clone(), self.options.clone(), child)
+                .map_err(|e| DataFusionError::External(e.into()))?;
+            Ok(Arc::new(rebuilt))
         }
     }
 
@@ -2058,10 +3030,16 @@ impl ExecutionPlan for FilteredReadExec {
         partition: usize,
         context: Arc<TaskContext>,
     ) -> DataFusionResult<SendableRecordBatchStream> {
-        Ok(self.obtain_stream(partition, context))
+        match &self.input {
+            RowSelector::RowStream(source) => self.execute_row_stream(source, partition, context),
+            _ => Ok(self.obtain_stream(partition, context)),
+        }
     }
 
     fn fetch(&self) -> Option<usize> {
+        if self.row_stream_input().is_some() {
+            return None;
+        }
         if self.options.full_filter.is_none() {
             self.options
                 .scan_range_before_filter
@@ -2076,13 +3054,15 @@ impl ExecutionPlan for FilteredReadExec {
     }
 
     fn supports_limit_pushdown(&self) -> bool {
-        // This is to push the limit through the node and into an upstream node.
-        // The only upstream node is the index search and we can't push the limit
-        // to that node.
-        false
+        // A limit pushes through to a row-stream input (one output row per
+        // input row); the other selectors have no node to push it to
+        self.row_stream_input().is_some()
     }
 
     fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
+        if self.row_stream_input().is_some() {
+            return None;
+        }
         // TODO: Support multiple partitions in the future by coordinating limits across partitions
         if matches!(
             self.options.threading_mode,
@@ -2113,7 +3093,7 @@ impl ExecutionPlan for FilteredReadExec {
         match Self::try_new(
             self.dataset.clone(),
             updated_options,
-            self.index_input.clone(),
+            self.input.row_set_plan().cloned(),
         ) {
             Ok(exec) => Some(Arc::new(exec)),
             Err(e) => {
@@ -2142,7 +3122,9 @@ mod tests {
     };
     use itertools::Itertools;
     use lance_core::datatypes::OnMissing;
+    use lance_core::utils::address::RowAddress;
     use lance_core::utils::tempfile::TempStrDir;
+    use lance_datafusion::exec::OneShotExec;
     use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch};
     use lance_index::{
         IndexType,
@@ -2150,6 +3132,7 @@ mod tests {
         scalar::{ScalarIndexParams, expression::PlannerIndexExt},
     };
     use lance_select::result::IndexExprResultWireFormat;
+    use lance_select::{RowAddrMask, RowAddrTreeMap};
 
     use crate::{
         dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams},
@@ -2376,6 +3359,77 @@ mod tests {
         ))
     }
 
+    /// Take-shaped masked reads consolidate their tiny per-fragment batches;
+    /// few-fragment and dense masked reads keep per-fragment boundaries.
+    #[test_log::test(tokio::test)]
+    async fn test_take_shaped_mask_consolidation() {
+        // 20 fragments x 2000 rows, value = global row number
+        let tmp_path = TempStrDir::default();
+        let data = gen_batch()
+            .col("value", array::step::<UInt32Type>())
+            .into_reader_rows(RowCount::from(2000), BatchCount::from(20));
+        let dataset = Arc::new(
+            Dataset::write(
+                data,
+                tmp_path.as_str(),
+                Some(WriteParams {
+                    max_rows_per_file: 2000,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let mask_input = |addrs: Vec<u64>| -> Arc<dyn ExecutionPlan> {
+            let covered: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect();
+            let batch =
+                IndexExprResult::exact(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(addrs)))
+                    .serialize(&covered, IndexExprResultWireFormat::default())
+                    .unwrap();
+            let schema = batch.schema();
+            let stream = futures::stream::once(async move { Ok(batch) });
+            Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new(
+                schema, stream,
+            ))))
+        };
+        let run = |input: Arc<dyn ExecutionPlan>| {
+            let dataset = dataset.clone();
+            async move {
+                // Pin the batch size so batch-count assertions don't depend
+                // on LANCE_DEFAULT_BATCH_SIZE
+                let options = FilteredReadOptions::basic_full_read(&dataset).with_batch_size(2000);
+                let plan =
+                    FilteredReadExec::try_new(dataset.clone(), options, Some(input)).unwrap();
+                let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+                stream.try_collect::<Vec<_>>().await.unwrap()
+            }
+        };
+        let addr = |frag: u32, offset: u32| u64::from(RowAddress::new_from_parts(frag, offset));
+
+        // Take shape: 20 fragments, 2 rows each -> one consolidated batch,
+        // rows in fragment order
+        let addrs: Vec<u64> = (0..20u32).flat_map(|f| [addr(f, 3), addr(f, 7)]).collect();
+        let batches = run(mask_input(addrs)).await;
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 40);
+        assert_eq!(batches.len(), 1);
+        let expected =
+            UInt32Array::from_iter_values((0..20u32).flat_map(|f| [f * 2000 + 3, f * 2000 + 7]));
+        assert_eq!(batches[0].column(0).as_ref(), &expected);
+
+        // Too few fragments -> inline path, one batch per fragment
+        let batches = run(mask_input(vec![addr(0, 3), addr(1, 7)])).await;
+        assert_eq!(batches.len(), 2);
+
+        // Dense (2000 planned rows per fragment) -> inline path
+        let addrs: Vec<u64> = (0..8u32)
+            .flat_map(|f| (0..2000u32).map(move |o| addr(f, o)))
+            .collect();
+        let batches = run(mask_input(addrs)).await;
+        assert_eq!(batches.len(), 8);
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 16000);
+    }
+
     /// Round-trip every interval shape through the arrow wire format and
     /// confirm the endpoints survive. Exercises both
     /// `IndexExprResult::serialize` and `EvaluatedIndex::try_from_arrow`
@@ -2410,15 +3464,15 @@ mod tests {
             // robust, but the canonical builders preserve representation.
             assert_eq!(
                 decoded.index_result.lower, original.lower,
-                "{name}: lower endpoint changed across round-trip",
+                "{name}: lower endpoint changed across batch-trip",
             );
             assert_eq!(
                 decoded.index_result.upper, original.upper,
-                "{name}: upper endpoint changed across round-trip",
+                "{name}: upper endpoint changed across batch-trip",
             );
             assert_eq!(
                 decoded.applicable_fragments, frags,
-                "{name}: applicable fragments changed across round-trip",
+                "{name}: applicable fragments changed across batch-trip",
             );
         }
     }
@@ -2765,6 +3819,48 @@ mod tests {
         assert_eq!(num_rows, 300);
     }
 
+    /// A stale (not rebuilt after a delete) index hit drops on the live view
+    /// and returns as a null-_rowid tombstone with with_deleted_rows
+    #[test_log::test(tokio::test)]
+    async fn test_with_deleted_rows_stale_index() {
+        let fixture = Arc::new(TestFixture::new().await);
+        let base_options = FilteredReadOptions::basic_full_read(&fixture.dataset);
+
+        // Row 220 is deletion-vector-deleted but still in the index
+        let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await;
+
+        // Live view: the stale index hit drops
+        fixture
+            .test_plan(
+                base_options.clone().with_filter_plan(filter_plan),
+                &u32s(vec![]),
+            )
+            .await;
+
+        // Physical view: the tombstone returns
+        let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await;
+        let options = base_options
+            .with_deleted_rows()
+            .unwrap()
+            .with_filter_plan(filter_plan);
+        let plan = fixture.make_plan(options).await;
+        let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total_rows, 1);
+        let batch = batches.iter().find(|b| b.num_rows() > 0).unwrap();
+        let values = batch
+            .column_by_name("fully_indexed")
+            .unwrap()
+            .as_primitive::<UInt32Type>();
+        assert_eq!(values.value(0), 220);
+        let row_ids = batch
+            .column_by_name(ROW_ID)
+            .unwrap()
+            .as_primitive::<UInt64Type>();
+        assert!(row_ids.is_null(0));
+    }
+
     #[test]
     fn test_dv_to_ranges() {
         let dv = Arc::new(DeletionVector::from_iter(vec![1]));
@@ -2967,10 +4063,7 @@ mod tests {
             assert_eq!(plan.options().scan_range_before_filter, None);
             assert_eq!(plan.fetch(), None);
             let new_plan = plan.with_fetch(Some(100)).unwrap();
-            let new_plan = new_plan
-                .as_any()
-                .downcast_ref::<FilteredReadExec>()
-                .unwrap();
+            let new_plan = new_plan.downcast_ref::<FilteredReadExec>().unwrap();
             assert_eq!(new_plan.options().scan_range_before_filter, Some(0..100));
             assert_eq!(new_plan.fetch(), Some(100));
         }
@@ -2998,10 +4091,7 @@ mod tests {
             assert_eq!(plan.options().scan_range_after_filter, None);
             assert_eq!(plan.fetch(), None);
             let new_plan = plan.with_fetch(Some(50)).unwrap();
-            let new_plan = new_plan
-                .as_any()
-                .downcast_ref::<FilteredReadExec>()
-                .unwrap();
+            let new_plan = new_plan.downcast_ref::<FilteredReadExec>().unwrap();
             assert_eq!(new_plan.options().scan_range_after_filter, Some(0..50));
             assert_eq!(new_plan.fetch(), Some(50));
         }
@@ -3051,14 +4141,11 @@ mod tests {
         let options = base_options.with_filter_plan(filter_plan);
         let plan = fixture.make_plan(options).await;
 
-        assert!(plan.index_input.is_some());
+        assert!(plan.index_input().is_some());
         assert!(plan.options().refine_filter.is_some());
 
         let limited_plan = plan.with_fetch(Some(10)).unwrap();
-        let limited_plan = limited_plan
-            .as_any()
-            .downcast_ref::<FilteredReadExec>()
-            .unwrap();
+        let limited_plan = limited_plan.downcast_ref::<FilteredReadExec>().unwrap();
         assert_eq!(limited_plan.options().scan_range_after_filter, Some(0..10));
 
         let stream = limited_plan
@@ -3780,7 +4867,7 @@ mod tests {
 
     /// Test that direct execution gives the same result as get_plan + execute_with_plan
     #[test_log::test(tokio::test)]
-    async fn test_plan_round_trip() {
+    async fn test_plan_batch_trip() {
         let fixture = TestFixture::new().await;
         let ctx = Arc::new(TaskContext::default());
 
@@ -3893,4 +4980,863 @@ mod tests {
 
         assert_eq!(default_result.num_rows(), capped_result.num_rows());
     }
+
+    // Row-stream selector tests
+
+    mod row_stream {
+        use super::*;
+        use arrow_array::{Float32Array, StringArray, UInt64Array};
+        use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
+        use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+        use lance_datafusion::exec::OneShotExec;
+        use rstest::rstest;
+
+        use crate::dataset::{Dataset, WriteParams};
+        use crate::utils::test::NoContextTestFixture;
+
+        struct TakeFixture {
+            dataset: Arc<Dataset>,
+            _tmp_dir: TempStrDir,
+        }
+
+        /// 30 rows across 3 fragments with columns i, s, and struct{x, y}
+        async fn take_fixture(stable_row_ids: bool) -> TakeFixture {
+            let struct_fields = Fields::from(vec![
+                Arc::new(ArrowField::new("x", DataType::Int32, false)),
+                Arc::new(ArrowField::new("y", DataType::Int32, false)),
+            ]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("i", DataType::Int32, false),
+                ArrowField::new("s", DataType::Utf8, false),
+                ArrowField::new("struct", DataType::Struct(struct_fields.clone()), false),
+            ]));
+            let batches: Vec<RecordBatch> = (0..3)
+                .map(|batch_id| {
+                    let value_range = batch_id * 10..batch_id * 10 + 10;
+                    RecordBatch::try_new(
+                        schema.clone(),
+                        vec![
+                            Arc::new(Int32Array::from_iter_values(value_range.clone())),
+                            Arc::new(StringArray::from_iter_values(
+                                value_range.clone().map(|v| format!("s-{v}")),
+                            )),
+                            Arc::new(arrow_array::StructArray::new(
+                                struct_fields.clone(),
+                                vec![
+                                    Arc::new(Int32Array::from_iter(value_range.clone())),
+                                    Arc::new(Int32Array::from_iter(value_range)),
+                                ],
+                                None,
+                            )),
+                        ],
+                    )
+                    .unwrap()
+                })
+                .collect();
+
+            let tmp_dir = TempStrDir::default();
+            let uri = tmp_dir.as_str();
+            let params = WriteParams {
+                max_rows_per_file: 10,
+                enable_stable_row_ids: stable_row_ids,
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+            Dataset::write(reader, uri, Some(params)).await.unwrap();
+            TakeFixture {
+                dataset: Arc::new(Dataset::open(uri).await.unwrap()),
+                _tmp_dir: tmp_dir,
+            }
+        }
+
+        /// Wrap batches of (payload, key) rows into an input plan
+        fn rows_input(batches: Vec<RecordBatch>) -> Arc<dyn ExecutionPlan> {
+            let schema = batches[0].schema();
+            let stream = futures::stream::iter(batches.into_iter().map(Ok));
+            let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+            Arc::new(OneShotExec::new(stream))
+        }
+
+        fn take_plan(
+            dataset: &Arc<Dataset>,
+            input: Arc<dyn ExecutionPlan>,
+            columns: &[&str],
+        ) -> Result<FilteredReadExec> {
+            let projection = dataset
+                .empty_projection()
+                .union_columns(columns, OnMissing::Error)
+                .unwrap();
+            FilteredReadExec::try_new(
+                dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(input),
+            )
+        }
+
+        fn take_plan_sized(
+            dataset: &Arc<Dataset>,
+            input: Arc<dyn ExecutionPlan>,
+            columns: &[&str],
+            batch_size: u32,
+        ) -> Result<FilteredReadExec> {
+            let projection = dataset
+                .empty_projection()
+                .union_columns(columns, OnMissing::Error)
+                .unwrap();
+            FilteredReadExec::try_new(
+                dataset.clone(),
+                FilteredReadOptions::new(projection).with_batch_size(batch_size),
+                Some(input),
+            )
+        }
+
+        async fn run(plan: &FilteredReadExec) -> Vec<RecordBatch> {
+            plan.execute(0, Arc::new(TaskContext::default()))
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap()
+        }
+
+        /// A sparse plan constructs fragment handles only for the fragments
+        /// it selects, keeping their candidate-list position as priority —
+        /// no metadata is loaded or retained for unselected fragments
+        #[tokio::test]
+        async fn sparse_plan_scopes_only_selected_fragments() {
+            let fixture = take_fixture(false).await;
+            let dataset = &fixture.dataset;
+            let descriptors = dataset.fragments().clone();
+            assert_eq!(descriptors.len(), 3);
+
+            let mut rows = BTreeMap::new();
+            rows.insert(2u32, vec![0u64..5]);
+            let plan = FilteredReadInternalPlan {
+                rows,
+                filters: HashMap::new(),
+                scan_range_after_filter: None,
+            };
+            let options = FilteredReadOptions::basic_full_read(dataset);
+            let scheduler = FilteredReadStream::make_scan_scheduler(dataset, &options);
+
+            let scoped = FilteredReadStream::plan_to_scoped_fragments(
+                &plan,
+                &descriptors,
+                dataset,
+                &options,
+                scheduler,
+            );
+            assert_eq!(scoped.len(), 1);
+            assert_eq!(scoped[0].fragment.id(), 2);
+            assert_eq!(scoped[0].priority, 2);
+        }
+
+        /// Output preserves the input's row order, duplicates, and payload
+        #[rstest]
+        #[case::by_row_addr(false, ROW_ADDR)]
+        #[case::by_row_id(false, ROW_ID)]
+        #[case::stable_by_row_addr(true, ROW_ADDR)]
+        #[case::stable_by_row_id(true, ROW_ID)]
+        #[tokio::test]
+        async fn take_preserves_order_dups_and_payload(
+            #[case] stable_row_ids: bool,
+            #[case] key: &str,
+        ) {
+            let fixture = take_fixture(stable_row_ids).await;
+
+            // Stable row ids are assigned sequentially on write, so the id of
+            // row `i` is `i`; without them id == address
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            let keys: Vec<u64> = if key == ROW_ID && stable_row_ids {
+                vec![21, 3, 15, 21, 0]
+            } else {
+                vec![
+                    addr(2, 1), // i = 21
+                    addr(0, 3), // i = 3
+                    addr(1, 5), // i = 15
+                    addr(2, 1), // i = 21 (duplicate)
+                    addr(0, 0), // i = 0
+                ]
+            };
+            let expected_i: Vec<i32> = vec![21, 3, 15, 21, 0];
+
+            let input_schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("payload", DataType::Float32, false),
+                ArrowField::new(key, DataType::UInt64, true),
+            ]));
+            let payload: Vec<f32> = (0..keys.len()).map(|v| v as f32 * 0.5).collect();
+            let batch = RecordBatch::try_new(
+                input_schema.clone(),
+                vec![
+                    Arc::new(Float32Array::from(payload.clone())),
+                    Arc::new(UInt64Array::from(keys.clone())),
+                ],
+            )
+            .unwrap();
+            let batches = vec![batch.slice(0, 3), batch.slice(3, 2)];
+
+            let plan = take_plan(&fixture.dataset, rows_input(batches), &["s", "i"]).unwrap();
+            assert!(plan.row_stream_input().is_some());
+            // Input columns, then new fields; the unrequested key is stripped
+            assert_eq!(
+                plan.schema()
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().clone())
+                    .collect::<Vec<_>>(),
+                vec!["payload", "i", "s"]
+            );
+
+            let result = run(&plan).await;
+            assert_eq!(result.len(), 1);
+            assert_eq!(result[0].num_rows(), 5);
+            let result = concat_batches(&plan.schema(), &result).unwrap();
+
+            let i_col = result.column_by_name("i").unwrap();
+            assert_eq!(
+                i_col.as_primitive::<arrow::datatypes::Int32Type>().values(),
+                &expected_i[..]
+            );
+            let s_col = result.column_by_name("s").unwrap().as_string::<i32>();
+            for (row, i) in expected_i.iter().enumerate() {
+                assert_eq!(s_col.value(row), format!("s-{i}"));
+            }
+            let payload_col = result
+                .column_by_name("payload")
+                .unwrap()
+                .as_primitive::<Float32Type>();
+            assert_eq!(payload_col.values(), &payload[..]);
+        }
+
+        /// Tiny input batches merge up to the target and oversized ones pass
+        /// through whole, preserving order across the boundaries
+        #[tokio::test]
+        async fn take_coalesces_input_to_batch_size() {
+            let fixture = take_fixture(false).await;
+
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            let keys: Vec<u64> = vec![
+                addr(2, 3), // i = 23
+                addr(0, 1), // i = 1
+                addr(1, 4), // i = 14
+                addr(0, 7), // i = 7
+                addr(2, 0), // i = 20
+                addr(1, 1), // i = 11
+                addr(0, 2), // i = 2
+            ];
+            let expected_i: Vec<i32> = vec![23, 1, 14, 7, 20, 11, 2];
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ADDR,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(
+                input_schema.clone(),
+                vec![Arc::new(UInt64Array::from(keys.clone()))],
+            )
+            .unwrap();
+
+            let assert_batches =
+                |result: Vec<RecordBatch>, schema: SchemaRef, expected_sizes: Vec<usize>| {
+                    assert_eq!(
+                        result.iter().map(|b| b.num_rows()).collect::<Vec<_>>(),
+                        expected_sizes
+                    );
+                    let merged = concat_batches(&schema, &result).unwrap();
+                    let i_col = merged.column_by_name("i").unwrap();
+                    assert_eq!(
+                        i_col.as_primitive::<arrow::datatypes::Int32Type>().values(),
+                        &expected_i[..]
+                    );
+                };
+
+            // Seven one-row batches merge whenever the buffer reaches 3 rows
+            let tiny = (0..7).map(|i| batch.slice(i, 1)).collect::<Vec<_>>();
+            let plan = take_plan_sized(&fixture.dataset, rows_input(tiny), &["i"], 3).unwrap();
+            assert_batches(run(&plan).await, plan.schema(), vec![3, 3, 1]);
+
+            // One oversized batch passes through whole — never split
+            let plan =
+                take_plan_sized(&fixture.dataset, rows_input(vec![batch.clone()]), &["i"], 3)
+                    .unwrap();
+            assert_batches(run(&plan).await, plan.schema(), vec![7]);
+
+            // A large batch flushes the partial buffer and passes through
+            let mixed = vec![batch.slice(0, 2), batch.slice(2, 5)];
+            let plan = take_plan_sized(&fixture.dataset, rows_input(mixed), &["i"], 3).unwrap();
+            assert_batches(run(&plan).await, plan.schema(), vec![2, 5]);
+        }
+
+        /// Storage-ordered input exercises the aligned fast path
+        #[rstest]
+        #[case::by_row_addr(ROW_ADDR)]
+        #[case::by_row_id(ROW_ID)]
+        #[tokio::test]
+        async fn take_aligned_input_fast_path(#[case] key: &str) {
+            let fixture = take_fixture(false).await;
+
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            let keys: Vec<u64> = vec![
+                addr(0, 0), // i = 0
+                addr(0, 3), // i = 3
+                addr(1, 5), // i = 15
+                addr(2, 1), // i = 21
+                addr(2, 9), // i = 29
+            ];
+            let expected_i: Vec<i32> = vec![0, 3, 15, 21, 29];
+
+            let input_schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("payload", DataType::Float32, false),
+                ArrowField::new(key, DataType::UInt64, true),
+            ]));
+            let payload: Vec<f32> = (0..keys.len()).map(|v| v as f32 * 0.5).collect();
+            let batch = RecordBatch::try_new(
+                input_schema,
+                vec![
+                    Arc::new(Float32Array::from(payload.clone())),
+                    Arc::new(UInt64Array::from(keys)),
+                ],
+            )
+            .unwrap();
+
+            let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap();
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            assert_eq!(result.num_rows(), 5);
+            let i_col = result.column_by_name("i").unwrap();
+            assert_eq!(
+                i_col.as_primitive::<arrow::datatypes::Int32Type>().values(),
+                &expected_i[..]
+            );
+            let payload_col = result
+                .column_by_name("payload")
+                .unwrap()
+                .as_primitive::<Float32Type>();
+            assert_eq!(payload_col.values(), &payload[..]);
+        }
+
+        /// A fragment-scoped take reads from the scoped fragments only; keys
+        /// pointing outside the scope drop like stale rows
+        #[rstest]
+        #[case::by_row_addr(false, ROW_ADDR)]
+        #[case::by_row_id(false, ROW_ID)]
+        #[case::stable_by_row_addr(true, ROW_ADDR)]
+        #[case::stable_by_row_id(true, ROW_ID)]
+        #[tokio::test]
+        async fn take_scoped_to_fragments(#[case] stable_row_ids: bool, #[case] key: &str) {
+            let fixture = take_fixture(stable_row_ids).await;
+            let subset = Arc::new(vec![fixture.dataset.fragments()[1].clone()]);
+
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            // i = 12 inside the scoped fragment, i = 3 outside the scope
+            let keys: Vec<u64> = if key == ROW_ID && stable_row_ids {
+                vec![12, 3]
+            } else {
+                vec![addr(1, 2), addr(0, 3)]
+            };
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                key,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))])
+                .unwrap();
+
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_columns(["i"], OnMissing::Error)
+                .unwrap();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection).with_fragments(subset),
+                Some(rows_input(vec![batch])),
+            )
+            .unwrap();
+
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            assert_eq!(result.num_rows(), 1);
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.value(0), 12);
+        }
+
+        /// A batch whose keys span the whole id range but hit only two rows:
+        /// the span prefilter must not misread coverage as membership
+        #[tokio::test]
+        async fn take_stable_ids_wide_key_span() {
+            let fixture = take_fixture(true).await;
+            // Last row of the last fragment, first row of the first: every
+            // fragment's span overlaps, only two rows match
+            let keys: Vec<u64> = vec![29, 0];
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ID,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))])
+                .unwrap();
+
+            let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap();
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.values(), &[29, 0]);
+        }
+
+        /// Identity flags: requested-but-missing columns are synthesized,
+        /// carried ones kept, unrequested carried ones stripped
+        #[rstest]
+        #[case::unstable(false)]
+        #[case::stable(true)]
+        #[tokio::test]
+        async fn take_identity_flags(#[case] stable_row_ids: bool) {
+            let fixture = take_fixture(stable_row_ids).await;
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+
+            let ids: Vec<u64> = if stable_row_ids {
+                vec![21, 3]
+            } else {
+                vec![addr(2, 1), addr(0, 3)]
+            };
+            let expected_addrs: Vec<u64> = vec![addr(2, 1), addr(0, 3)];
+            let expected_i: Vec<i32> = vec![21, 3];
+            let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ID,
+                DataType::UInt64,
+                true,
+            )]));
+            let id_batch =
+                RecordBatch::try_new(id_schema, vec![Arc::new(UInt64Array::from(ids.clone()))])
+                    .unwrap();
+
+            // Keep the carried _rowid and synthesize _rowaddr
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_columns(["i"], OnMissing::Error)
+                .unwrap()
+                .with_row_id()
+                .with_row_addr();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(rows_input(vec![id_batch.clone()])),
+            )
+            .unwrap();
+            assert_eq!(
+                plan.schema()
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().clone())
+                    .collect::<Vec<_>>(),
+                vec![ROW_ID, "i", ROW_ADDR]
+            );
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let id_col = result
+                .column_by_name(ROW_ID)
+                .unwrap()
+                .as_primitive::<arrow::datatypes::UInt64Type>();
+            assert_eq!(id_col.values(), &ids[..]);
+            let addr_col = result
+                .column_by_name(ROW_ADDR)
+                .unwrap()
+                .as_primitive::<arrow::datatypes::UInt64Type>();
+            assert_eq!(addr_col.values(), &expected_addrs[..]);
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.values(), &expected_i[..]);
+
+            // Synthesize _rowaddr but strip the unrequested carried _rowid
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_columns(["i"], OnMissing::Error)
+                .unwrap()
+                .with_row_addr();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(rows_input(vec![id_batch.clone()])),
+            )
+            .unwrap();
+            assert_eq!(
+                plan.schema()
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().clone())
+                    .collect::<Vec<_>>(),
+                vec!["i", ROW_ADDR]
+            );
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let addr_col = result
+                .column_by_name(ROW_ADDR)
+                .unwrap()
+                .as_primitive::<arrow::datatypes::UInt64Type>();
+            assert_eq!(addr_col.values(), &expected_addrs[..]);
+
+            // Address-keyed input, synthesize _rowid (the reverse direction)
+            let addr_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ADDR,
+                DataType::UInt64,
+                true,
+            )]));
+            let addr_batch = RecordBatch::try_new(
+                addr_schema,
+                vec![Arc::new(UInt64Array::from(expected_addrs.clone()))],
+            )
+            .unwrap();
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_columns(["i"], OnMissing::Error)
+                .unwrap()
+                .with_row_id();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(rows_input(vec![addr_batch])),
+            )
+            .unwrap();
+            assert_eq!(
+                plan.schema()
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().clone())
+                    .collect::<Vec<_>>(),
+                vec!["i", ROW_ID]
+            );
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let id_col = result
+                .column_by_name(ROW_ID)
+                .unwrap()
+                .as_primitive::<arrow::datatypes::UInt64Type>();
+            assert_eq!(id_col.values(), &ids[..]);
+
+            // Fetch nothing, synthesize only (the AddRowAddrExec shape)
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .with_row_id()
+                .with_row_addr();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(rows_input(vec![id_batch])),
+            )
+            .unwrap();
+            assert_eq!(
+                plan.schema()
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().clone())
+                    .collect::<Vec<_>>(),
+                vec![ROW_ID, ROW_ADDR]
+            );
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let addr_col = result
+                .column_by_name(ROW_ADDR)
+                .unwrap()
+                .as_primitive::<arrow::datatypes::UInt64Type>();
+            assert_eq!(addr_col.values(), &expected_addrs[..]);
+        }
+
+        /// New sub-fields merge into an existing struct column
+        #[tokio::test]
+        async fn take_merges_nested_struct() {
+            let fixture = take_fixture(false).await;
+
+            let data = fixture
+                .dataset
+                .scan()
+                .project(&["struct"])
+                .unwrap()
+                .with_row_id()
+                .try_into_batch()
+                .await
+                .unwrap();
+            // Rebuild the input with only struct.y so struct.x must be taken
+            let full_struct = data.column_by_name("struct").unwrap().as_struct();
+            let y_only = arrow_array::StructArray::new(
+                Fields::from(vec![Arc::new(ArrowField::new("y", DataType::Int32, false))]),
+                vec![full_struct.column_by_name("y").unwrap().clone()],
+                None,
+            );
+            let input_schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("struct", y_only.data_type().clone(), false),
+                ArrowField::new(ROW_ID, DataType::UInt64, true),
+            ]));
+            let data = RecordBatch::try_new(
+                input_schema,
+                vec![
+                    Arc::new(y_only),
+                    data.column_by_name(ROW_ID).unwrap().clone(),
+                ],
+            )
+            .unwrap();
+
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_column("struct.x", OnMissing::Error)
+                .unwrap();
+            let plan = FilteredReadExec::try_new(
+                fixture.dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(rows_input(vec![data])),
+            )
+            .unwrap();
+
+            let expected_struct_type = DataType::Struct(Fields::from(vec![
+                Arc::new(ArrowField::new("x", DataType::Int32, false)),
+                Arc::new(ArrowField::new("y", DataType::Int32, false)),
+            ]));
+            assert_eq!(
+                plan.schema().field_with_name("struct").unwrap().data_type(),
+                &expected_struct_type
+            );
+
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            assert_eq!(result.num_rows(), 30);
+            let struct_col = result.column_by_name("struct").unwrap().as_struct();
+            assert_eq!(
+                struct_col.column_by_name("x").unwrap(),
+                struct_col.column_by_name("y").unwrap()
+            );
+        }
+
+        /// Input rows whose key no longer exists (deleted rows) are dropped
+        #[rstest]
+        #[case::by_row_addr(false, ROW_ADDR)]
+        #[case::by_row_id(false, ROW_ID)]
+        #[case::stable_by_row_addr(true, ROW_ADDR)]
+        #[case::stable_by_row_id(true, ROW_ID)]
+        #[tokio::test]
+        async fn take_drops_stale_keys(#[case] stable_row_ids: bool, #[case] key: &str) {
+            let fixture = take_fixture(stable_row_ids).await;
+            let mut dataset = fixture.dataset.as_ref().clone();
+            dataset.delete("i = 15").await.unwrap();
+            let dataset = Arc::new(dataset);
+
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            // The pre-delete identifiers of rows 15 (now deleted) and 16
+            let keys: Vec<u64> = if key == ROW_ID && stable_row_ids {
+                vec![15, 16]
+            } else {
+                vec![addr(1, 5), addr(1, 6)]
+            };
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                key,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))])
+                .unwrap();
+
+            let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap();
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            assert_eq!(result.num_rows(), 1);
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.value(0), 16);
+        }
+
+        /// Keys of a fully deleted fragment (gone from the manifest) are
+        /// dropped like stale rows
+        #[rstest]
+        #[case::by_row_addr(false, ROW_ADDR)]
+        #[case::by_row_id(false, ROW_ID)]
+        #[case::stable_by_row_addr(true, ROW_ADDR)]
+        #[case::stable_by_row_id(true, ROW_ID)]
+        #[tokio::test]
+        async fn take_drops_keys_of_deleted_fragment(
+            #[case] stable_row_ids: bool,
+            #[case] key: &str,
+        ) {
+            let fixture = take_fixture(stable_row_ids).await;
+            let mut dataset = fixture.dataset.as_ref().clone();
+            dataset.delete("i >= 10 and i < 20").await.unwrap();
+            let dataset = Arc::new(dataset);
+
+            let addr = |frag: u64, off: u64| (frag << 32) | off;
+            // The pre-delete identifiers of row 15 (fragment 1, now gone
+            // from the manifest) and row 20 (fragment 2, still live)
+            let keys: Vec<u64> = if key == ROW_ID && stable_row_ids {
+                vec![15, 20]
+            } else {
+                vec![addr(1, 5), addr(2, 0)]
+            };
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                key,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))])
+                .unwrap();
+
+            let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap();
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            assert_eq!(result.num_rows(), 1);
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.value(0), 20);
+        }
+
+        /// After delete + compaction the stable row-id sequences are no
+        /// longer simple contiguous ranges; ids must still resolve to the
+        /// moved rows and deleted ids must still drop
+        #[tokio::test]
+        async fn take_stable_ids_after_compaction() {
+            use crate::dataset::optimize::{CompactionOptions, compact_files};
+
+            let fixture = take_fixture(true).await;
+            let mut dataset = fixture.dataset.as_ref().clone();
+            // Punch holes, then rewrite all fragments into one
+            dataset.delete("i % 3 = 0").await.unwrap();
+            compact_files(&mut dataset, CompactionOptions::default(), None)
+                .await
+                .unwrap();
+            let dataset = Arc::new(dataset);
+            assert_eq!(dataset.get_fragments().len(), 1);
+
+            // Survivors in scattered order, plus a compacted-away id (15)
+            let keys: Vec<u64> = vec![25, 1, 15, 14];
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ID,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))])
+                .unwrap();
+
+            let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap();
+            let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap();
+            let i_col = result
+                .column_by_name("i")
+                .unwrap()
+                .as_primitive::<arrow::datatypes::Int32Type>();
+            assert_eq!(i_col.values(), &[25, 1, 14]);
+        }
+
+        /// with_deleted_rows is rejected for a row-stream read
+        #[rstest]
+        #[case::unstable(false)]
+        #[case::stable(true)]
+        #[tokio::test]
+        async fn take_rejects_with_deleted_rows(#[case] stable_row_ids: bool) {
+            let fixture = take_fixture(stable_row_ids).await;
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ADDR,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch =
+                RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(vec![0_u64]))])
+                    .unwrap();
+            let projection = fixture
+                .dataset
+                .empty_projection()
+                .union_columns(["i"], OnMissing::Error)
+                .unwrap();
+            let err = FilteredReadExec::try_new(
+                fixture.dataset,
+                FilteredReadOptions::new(projection)
+                    .with_deleted_rows()
+                    .unwrap(),
+                Some(rows_input(vec![batch])),
+            )
+            .unwrap_err();
+            assert!(matches!(err, Error::InvalidInput { .. }), "{err}");
+            assert!(err.to_string().contains("with_deleted_rows"));
+        }
+
+        /// Construction errors: no key column, nothing to read
+        #[tokio::test]
+        async fn take_construction_errors() {
+            let fixture = take_fixture(false).await;
+            let no_key_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                "payload",
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(
+                no_key_schema,
+                vec![Arc::new(UInt64Array::from(vec![0_u64]))],
+            )
+            .unwrap();
+            let err = take_plan(&fixture.dataset, rows_input(vec![batch]), &["s"]).unwrap_err();
+            assert!(matches!(err, Error::InvalidInput { .. }), "{err}");
+            assert!(err.to_string().contains("must have a column"));
+
+            // Taking fields the input already has: nothing to read
+            let with_s_schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new(ROW_ADDR, DataType::UInt64, true),
+                ArrowField::new("s", DataType::Utf8, false),
+            ]));
+            let with_s_batch = RecordBatch::try_new(
+                with_s_schema,
+                vec![
+                    Arc::new(UInt64Array::from(vec![0_u64])),
+                    Arc::new(StringArray::from(vec!["x"])),
+                ],
+            )
+            .unwrap();
+            let err =
+                take_plan(&fixture.dataset, rows_input(vec![with_s_batch]), &["s"]).unwrap_err();
+            assert!(matches!(err, Error::InvalidInput { .. }), "{err}");
+            assert!(err.to_string().contains("nothing to read"));
+        }
+
+        /// with_new_children re-derives the row-stream source and preserves the schema
+        #[tokio::test]
+        async fn take_with_new_children() {
+            let fixture = take_fixture(false).await;
+            let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                ROW_ID,
+                DataType::UInt64,
+                true,
+            )]));
+            let batch = RecordBatch::try_new(
+                input_schema,
+                vec![Arc::new(UInt64Array::from(vec![0_u64, 1]))],
+            )
+            .unwrap();
+            let input = rows_input(vec![batch]);
+            let plan: Arc<dyn ExecutionPlan> =
+                Arc::new(take_plan(&fixture.dataset, input.clone(), &["s"]).unwrap());
+            let rebuilt = plan.clone().with_new_children(vec![input]).unwrap();
+            assert_eq!(plan.schema(), rebuilt.schema());
+            assert!(
+                rebuilt
+                    .downcast_ref::<FilteredReadExec>()
+                    .unwrap()
+                    .row_stream_input()
+                    .is_some()
+            );
+        }
+
+        /// Take-mode nodes can be created and executed without an active
+        /// tokio runtime (required for DataFusion foreign table providers)
+        #[test]
+        fn no_context_take_rows() {
+            use lance_datafusion::datagen::DatafusionDatagenExt;
+            use lance_datagen::{BatchCount, RowCount};
+
+            let fixture = NoContextTestFixture::new();
+            let dataset = Arc::new(fixture.dataset);
+            let input = lance_datagen::gen_batch()
+                .col(ROW_ID, lance_datagen::array::step::<UInt64Type>())
+                .into_df_exec(RowCount::from(50), BatchCount::from(2));
+            let plan = take_plan(&dataset, input, &["text"]).unwrap();
+            plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+        }
+    }
 }
diff --git a/rust/lance/src/io/exec/filtered_read_proto.rs b/rust/lance/src/io/exec/filtered_read_proto.rs
index 4eb329506aa..f351e42b1a6 100644
--- a/rust/lance/src/io/exec/filtered_read_proto.rs
+++ b/rust/lance/src/io/exec/filtered_read_proto.rs
@@ -727,6 +727,59 @@ mod tests {
         );
     }
 
+    /// A row-stream (take) exec serializes like any other: the input plan
+    /// travels as the node's child through the plan codec, and decoding
+    /// re-derives the row-stream selector from the child's schema
+    #[tokio::test]
+    async fn test_exec_to_proto_roundtrip_row_stream() {
+        use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+        use lance_core::{ROW_ID, ROW_ID_FIELD};
+        use lance_datafusion::exec::OneShotExec;
+
+        fn keys_input() -> Arc<dyn ExecutionPlan> {
+            let schema = Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()]));
+            let batch = arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::UInt64Array::from(vec![3u64, 1, 4]))],
+            )
+            .unwrap();
+            let stream = futures::stream::iter(vec![Ok(batch)]);
+            Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new(
+                schema, stream,
+            ))))
+        }
+
+        let dataset = make_test_dataset().await;
+        let ctx = SessionContext::new();
+        let state = ctx.state();
+
+        let options = FilteredReadOptions::basic_full_read(&dataset);
+        let exec = FilteredReadExec::try_new(dataset.clone(), options, Some(keys_input())).unwrap();
+        assert!(exec.row_stream_input().is_some());
+
+        let proto = filtered_read_exec_to_proto(&exec, &state).await.unwrap();
+
+        // The codec hands the decoded child back; the selector re-derives
+        // from its schema
+        let back =
+            filtered_read_exec_from_proto(proto, Some(dataset.clone()), Some(keys_input()), &state)
+                .await
+                .unwrap();
+        assert!(back.row_stream_input().is_some());
+        assert_eq!(exec.schema(), back.schema());
+        assert_eq!(
+            exec.options().projection.field_ids,
+            back.options().projection.field_ids
+        );
+        assert!(
+            back.row_stream_input()
+                .unwrap()
+                .schema()
+                .column_with_name(ROW_ID)
+                .is_some()
+        );
+    }
+
     #[tokio::test]
     async fn test_plan_proto_roundtrip() {
         let dataset = make_test_dataset().await;
diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs
index add864c0ea9..f67ed5017c8 100644
--- a/rust/lance/src/io/exec/fts.rs
+++ b/rust/lance/src/io/exec/fts.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
 use arrow::array::{AsArray, BooleanBuilder};
@@ -13,7 +13,7 @@ use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::empty::EmptyExec;
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
-use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, Gauge, MetricsSet};
 use datafusion::physical_plan::repartition::RepartitionExec;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::union::UnionExec;
@@ -21,7 +21,7 @@ use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, Pla
 use datafusion_physical_expr::expressions::Column;
 use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning};
 use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
-use datafusion_physical_plan::metrics::{BaselineMetrics, Count};
+use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Time};
 use futures::future::try_join_all;
 use futures::stream::{self};
 use futures::{FutureExt, StreamExt, TryStreamExt};
@@ -51,11 +51,12 @@ use lance_index::scalar::inverted::query::{
 use lance_index::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer;
 use lance_index::scalar::inverted::{
     FTS_SCHEMA, InvertedIndex, MemBM25Scorer, SCORE_COL, build_global_bm25_scorer,
-    flat_bm25_search_stream_with_metrics,
+    flat_bm25_search_stream_with_metrics_and_operator,
 };
 use lance_index::{prefilter::PreFilter, scalar::inverted::query::BooleanQuery};
 use lance_tokenizer::{SimpleTokenizer, TextAnalyzer};
 use tracing::instrument;
+use uuid::Uuid;
 
 /// Open one FTS segment as an [`InvertedIndex`].
 async fn open_fts_segment(
@@ -135,7 +136,7 @@ async fn search_segments(
     let mut searches = searches;
 
     while let Some((doc_ids, scores)) = searches.try_next().await? {
-        for (row_id, score) in doc_ids.into_iter().zip(scores.into_iter()) {
+        for (row_id, score) in doc_ids.into_iter().zip(scores) {
             if candidates.len() < limit {
                 candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score)));
             } else if candidates.peek().unwrap().0.score.0 < score {
@@ -159,6 +160,105 @@ fn default_text_tokenizer() -> Box<dyn LanceTokenizer> {
     ))
 }
 
+/// Time spent resolving an exact ordered UUID selection to committed FTS segments.
+pub const FTS_SEGMENT_BIND_DURATION_METRIC: &str = "fts_segment_bind_duration";
+
+#[derive(Debug, Clone)]
+enum FtsSegmentSelection {
+    AllCommitted,
+    ExactResolved(Arc<[IndexMetadata]>),
+    ExactUuids(Arc<[Uuid]>),
+}
+
+impl FtsSegmentSelection {
+    fn exact_uuids(mut uuids: Vec<Uuid>) -> Self {
+        let mut seen = HashSet::with_capacity(uuids.len());
+        uuids.retain(|uuid| seen.insert(*uuid));
+        Self::ExactUuids(Arc::from(uuids))
+    }
+
+    fn preset_segments(&self) -> Option<&[IndexMetadata]> {
+        match self {
+            Self::ExactResolved(segments) => Some(segments),
+            Self::AllCommitted | Self::ExactUuids(_) => None,
+        }
+    }
+
+    fn explicit_segment_uuids(&self) -> Option<Vec<Uuid>> {
+        match self {
+            Self::AllCommitted => None,
+            Self::ExactResolved(segments) => {
+                Some(segments.iter().map(|segment| segment.uuid).collect())
+            }
+            Self::ExactUuids(uuids) => Some(uuids.to_vec()),
+        }
+    }
+
+    async fn resolve(
+        &self,
+        dataset: &Dataset,
+        column: &str,
+        segment_bind_duration: &Time,
+    ) -> DataFusionResult<Arc<[IndexMetadata]>> {
+        match self {
+            Self::AllCommitted => load_segments(dataset, column)
+                .await?
+                .map(Arc::from)
+                .ok_or_else(|| {
+                    DataFusionError::Execution(format!(
+                        "No Inverted index found for column {}",
+                        column,
+                    ))
+                }),
+            Self::ExactResolved(segments) => Ok(segments.clone()),
+            Self::ExactUuids(uuids) => {
+                let _timer = segment_bind_duration.timer();
+                let dataset_version = dataset.version_id();
+                if uuids.is_empty() {
+                    return Err(DataFusionError::Execution(format!(
+                        "Exact FTS segment selection for column {} at dataset version {} \
+                         requires at least one segment UUID",
+                        column, dataset_version
+                    )));
+                }
+
+                let committed_segments =
+                    load_segments(dataset, column).await?.ok_or_else(|| {
+                        DataFusionError::Execution(format!(
+                            "Cannot resolve exact FTS segment selection for column {} at dataset \
+                             version {}: no Inverted index found",
+                            column, dataset_version
+                        ))
+                    })?;
+                let mut segments_by_uuid = HashMap::with_capacity(committed_segments.len());
+                for segment in committed_segments {
+                    let uuid = segment.uuid;
+                    if segments_by_uuid.insert(uuid, segment).is_some() {
+                        return Err(DataFusionError::Execution(format!(
+                            "FTS metadata for column {} at dataset version {} contains duplicate \
+                             segment UUID {}",
+                            column, dataset_version, uuid
+                        )));
+                    }
+                }
+
+                let mut resolved = Vec::with_capacity(uuids.len());
+                for uuid in uuids.iter() {
+                    let segment = segments_by_uuid.get(uuid).ok_or_else(|| {
+                        DataFusionError::Execution(format!(
+                            "Requested FTS segment UUID {} for column {} is not committed in \
+                             dataset version {}",
+                            uuid, column, dataset_version
+                        ))
+                    })?;
+                    resolved.push(segment.clone());
+                }
+                Ok(Arc::from(resolved))
+            }
+        }
+    }
+}
+
 pub struct FtsIndexMetrics {
     index_metrics: IndexMetrics,
     partitions_searched: Count,
@@ -166,6 +266,10 @@ pub struct FtsIndexMetrics {
     and_candidates_pruned_before_return: Count,
     and_full_scores: Count,
     freqs_collected: Count,
+    /// Wall time (ms) of the exec-local `build_global_bm25_scorer`
+    /// fallback; zero when a preset base scorer was injected.
+    scorer_build_ms: Gauge,
+    segment_bind_duration: Time,
     baseline_metrics: BaselineMetrics,
 }
 
@@ -179,6 +283,8 @@ impl FtsIndexMetrics {
                 .new_count(AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, partition),
             and_full_scores: metrics.new_count(AND_FULL_SCORES_METRIC, partition),
             freqs_collected: metrics.new_count(FREQS_COLLECTED_METRIC, partition),
+            scorer_build_ms: metrics.new_gauge("scorer_build_ms", partition),
+            segment_bind_duration: metrics.new_time(FTS_SEGMENT_BIND_DURATION_METRIC, partition),
             baseline_metrics: BaselineMetrics::new(metrics, partition),
         }
     }
@@ -186,6 +292,10 @@ impl FtsIndexMetrics {
     pub fn record_parts_searched(&self, num_parts: usize) {
         self.partitions_searched.add(num_parts);
     }
+
+    pub fn record_scorer_build(&self, elapsed: std::time::Duration) {
+        self.scorer_build_ms.set(elapsed.as_millis() as usize);
+    }
 }
 
 impl MetricsCollector for FtsIndexMetrics {
@@ -227,9 +337,7 @@ pub struct MatchQueryExec {
     /// When set, `execute()` skips `build_global_bm25_scorer` and threads this
     /// scorer down to `InvertedIndex::bm25_search`.
     base_scorer: Option<Arc<MemBM25Scorer>>,
-    /// When set, `execute()` skips `load_segments` and searches exactly these
-    /// segments.
-    preset_segments: Option<Vec<IndexMetadata>>,
+    segment_selection: FtsSegmentSelection,
 
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
@@ -241,7 +349,7 @@ impl DisplayAs for MatchQueryExec {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
                 write!(
                     f,
-                    "MatchQuery: column={}, query={}",
+                    "MatchQuery: column={}, query=[{}]",
                     self.query.column.as_deref().unwrap_or_default(),
                     self.query.terms
                 )
@@ -287,7 +395,7 @@ impl MatchQueryExec {
             params,
             prefilter_source,
             base_scorer: None,
-            preset_segments: None,
+            segment_selection: FtsSegmentSelection::AllCommitted,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         }
@@ -322,7 +430,40 @@ impl MatchQueryExec {
             params,
             prefilter_source,
             base_scorer: None,
-            preset_segments: Some(segments),
+            segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)),
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+        }
+    }
+
+    /// Construct a `MatchQueryExec` bound to an exact ordered set of committed
+    /// FTS segment UUIDs.
+    ///
+    /// The UUIDs are resolved from this exec's dataset snapshot when the output
+    /// stream is polled. Duplicate UUIDs are removed while preserving their
+    /// first-occurrence order. Resolution fails if the list is empty or any UUID
+    /// is not committed for the query column.
+    pub fn new_with_segment_uuids(
+        dataset: Arc<Dataset>,
+        query: MatchQuery,
+        params: FtsSearchParams,
+        prefilter_source: PreFilterSource,
+        segment_uuids: Vec<Uuid>,
+    ) -> Self {
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(FTS_SCHEMA.clone()),
+            Partitioning::RoundRobinBatch(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        let params = Self::effective_params(&query, params);
+        Self {
+            dataset,
+            query,
+            params,
+            prefilter_source,
+            base_scorer: None,
+            segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids),
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         }
@@ -366,7 +507,16 @@ impl MatchQueryExec {
     }
 
     pub fn preset_segments(&self) -> Option<&[IndexMetadata]> {
-        self.preset_segments.as_deref()
+        self.segment_selection.preset_segments()
+    }
+
+    /// Return the ordered segment UUIDs for an explicit selection.
+    ///
+    /// Returns `None` when this exec searches all committed segments. UUID-based
+    /// selections omit duplicates while preserving first-occurrence order.
+    /// Pre-resolved selections preserve the supplied metadata order.
+    pub fn explicit_segment_uuids(&self) -> Option<Vec<Uuid>> {
+        self.segment_selection.explicit_segment_uuids()
     }
 }
 
@@ -375,10 +525,6 @@ impl ExecutionPlan for MatchQueryExec {
         "MatchQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.prefilter_source {
             PreFilterSource::None => vec![],
@@ -413,7 +559,7 @@ impl ExecutionPlan for MatchQueryExec {
                     params: self.params.clone(),
                     prefilter_source: PreFilterSource::None,
                     base_scorer: self.base_scorer.clone(),
-                    preset_segments: self.preset_segments.clone(),
+                    segment_selection: self.segment_selection.clone(),
                     properties: self.properties.clone(),
                     metrics: ExecutionPlanMetricsSet::new(),
                 }
@@ -440,7 +586,7 @@ impl ExecutionPlan for MatchQueryExec {
                     params: self.params.clone(),
                     prefilter_source,
                     base_scorer: self.base_scorer.clone(),
-                    preset_segments: self.preset_segments.clone(),
+                    segment_selection: self.segment_selection.clone(),
                     properties: self.properties.clone(),
                     metrics: ExecutionPlanMetricsSet::new(),
                 }
@@ -465,7 +611,7 @@ impl ExecutionPlan for MatchQueryExec {
         let ds = self.dataset.clone();
         let prefilter_source = self.prefilter_source.clone();
         let preset_base_scorer = self.base_scorer.clone();
-        let preset_segments = self.preset_segments.clone();
+        let segment_selection = self.segment_selection.clone();
         let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition));
         let column = query.column.ok_or(DataFusionError::Execution(format!(
             "column not set for MatchQuery {}",
@@ -473,15 +619,9 @@ impl ExecutionPlan for MatchQueryExec {
         )))?;
         let stream = stream::once(async move {
             let _timer = metrics.baseline_metrics.elapsed_compute().timer();
-            let segments = match preset_segments {
-                Some(segments) => segments,
-                None => load_segments(&ds, &column)
-                    .await?
-                    .ok_or(DataFusionError::Execution(format!(
-                        "No Inverted index found for column {}",
-                        column,
-                    )))?,
-            };
+            let segments = segment_selection
+                .resolve(&ds, &column, &metrics.segment_bind_duration)
+                .await?;
             let _details = load_segment_details(&ds, &column, &segments).await?;
             let indices =
                 open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?;
@@ -525,11 +665,16 @@ impl ExecutionPlan for MatchQueryExec {
             let tokens = collect_query_tokens(&query.terms, &mut tokenizer);
             let base_scorer = match preset_base_scorer {
                 Some(scorer) => scorer,
-                None => Arc::new(
-                    build_global_bm25_scorer(&indices, &tokens, &params)
-                        .boxed()
-                        .await?,
-                ),
+                None => {
+                    let scorer_start = std::time::Instant::now();
+                    let scorer = Arc::new(
+                        build_global_bm25_scorer(&indices, &tokens, &params)
+                            .boxed()
+                            .await?,
+                    );
+                    metrics.record_scorer_build(scorer_start.elapsed());
+                    scorer
+                }
             };
 
             pre_filter.wait_for_ready().await?;
@@ -795,10 +940,6 @@ impl ExecutionPlan for FlatMatchFilterExec {
         "FlatMatchFilterExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.input]
     }
@@ -850,7 +991,7 @@ impl ExecutionPlan for FlatMatchFilterExec {
         Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
@@ -991,10 +1132,6 @@ impl ExecutionPlan for FlatMatchQueryExec {
         "FlatMatchQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.unindexed_input]
     }
@@ -1078,13 +1215,16 @@ impl ExecutionPlan for FlatMatchQueryExec {
                         Some(scorer) => (*scorer).clone(),
                         None => {
                             let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer);
-                            build_global_bm25_scorer(
+                            let scorer_start = std::time::Instant::now();
+                            let scorer = build_global_bm25_scorer(
                                 &indices,
                                 &query_tokens,
                                 &FtsSearchParams::new(),
                             )
                             .boxed()
-                            .await?
+                            .await?;
+                            metrics.record_scorer_build(scorer_start.elapsed());
+                            scorer
                         }
                     };
                     (tokenizer, Some(base_scorer))
@@ -1095,13 +1235,14 @@ impl ExecutionPlan for FlatMatchQueryExec {
                 ),
             };
 
-            flat_bm25_search_stream_with_metrics(
+            flat_bm25_search_stream_with_metrics_and_operator(
                 unindexed_input,
                 column,
                 query.terms,
                 tokenizer,
                 base_scorer,
                 target_batch_size,
+                query.operator,
                 Some(elapsed_compute),
             )
             .await
@@ -1147,9 +1288,7 @@ pub struct PhraseQueryExec {
     /// Optional override for the BM25 scorer normally built locally inside
     /// `execute()`. See [`MatchQueryExec::with_base_scorer`].
     base_scorer: Option<Arc<MemBM25Scorer>>,
-    /// Optional pre-resolved segment list. See
-    /// [`MatchQueryExec::new_with_segments`].
-    preset_segments: Option<Vec<IndexMetadata>>,
+    segment_selection: FtsSegmentSelection,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
 }
@@ -1198,7 +1337,7 @@ impl PhraseQueryExec {
             params,
             prefilter_source,
             base_scorer: None,
-            preset_segments: None,
+            segment_selection: FtsSegmentSelection::AllCommitted,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         }
@@ -1226,7 +1365,41 @@ impl PhraseQueryExec {
             params,
             prefilter_source,
             base_scorer: None,
-            preset_segments: Some(segments),
+            segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)),
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+        }
+    }
+
+    /// Construct a `PhraseQueryExec` bound to an exact ordered set of committed
+    /// FTS segment UUIDs.
+    ///
+    /// The UUIDs are resolved from this exec's dataset snapshot when the output
+    /// stream is polled. Duplicate UUIDs are removed while preserving their
+    /// first-occurrence order. Resolution fails if the list is empty or any UUID
+    /// is not committed for the query column.
+    pub fn new_with_segment_uuids(
+        dataset: Arc<Dataset>,
+        query: PhraseQuery,
+        mut params: FtsSearchParams,
+        prefilter_source: PreFilterSource,
+        segment_uuids: Vec<Uuid>,
+    ) -> Self {
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(FTS_SCHEMA.clone()),
+            Partitioning::RoundRobinBatch(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        params = params.with_phrase_slop(Some(query.slop));
+
+        Self {
+            dataset,
+            query,
+            params,
+            prefilter_source,
+            base_scorer: None,
+            segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids),
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         }
@@ -1259,7 +1432,16 @@ impl PhraseQueryExec {
     }
 
     pub fn preset_segments(&self) -> Option<&[IndexMetadata]> {
-        self.preset_segments.as_deref()
+        self.segment_selection.preset_segments()
+    }
+
+    /// Return the ordered segment UUIDs for an explicit selection.
+    ///
+    /// Returns `None` when this exec searches all committed segments. UUID-based
+    /// selections omit duplicates while preserving first-occurrence order.
+    /// Pre-resolved selections preserve the supplied metadata order.
+    pub fn explicit_segment_uuids(&self) -> Option<Vec<Uuid>> {
+        self.segment_selection.explicit_segment_uuids()
     }
 }
 
@@ -1268,10 +1450,6 @@ impl ExecutionPlan for PhraseQueryExec {
         "PhraseQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.prefilter_source {
             PreFilterSource::None => vec![],
@@ -1299,7 +1477,7 @@ impl ExecutionPlan for PhraseQueryExec {
                 params: self.params.clone(),
                 prefilter_source: PreFilterSource::None,
                 base_scorer: self.base_scorer.clone(),
-                preset_segments: self.preset_segments.clone(),
+                segment_selection: self.segment_selection.clone(),
                 properties: self.properties.clone(),
                 metrics: ExecutionPlanMetricsSet::new(),
             },
@@ -1324,7 +1502,7 @@ impl ExecutionPlan for PhraseQueryExec {
                     params: self.params.clone(),
                     prefilter_source,
                     base_scorer: self.base_scorer.clone(),
-                    preset_segments: self.preset_segments.clone(),
+                    segment_selection: self.segment_selection.clone(),
                     properties: self.properties.clone(),
                     metrics: ExecutionPlanMetricsSet::new(),
                 }
@@ -1349,7 +1527,7 @@ impl ExecutionPlan for PhraseQueryExec {
         let ds = self.dataset.clone();
         let prefilter_source = self.prefilter_source.clone();
         let preset_base_scorer = self.base_scorer.clone();
-        let preset_segments = self.preset_segments.clone();
+        let segment_selection = self.segment_selection.clone();
         let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition));
         let stream = stream::once(async move {
             let _timer = metrics.baseline_metrics.elapsed_compute().timer();
@@ -1357,15 +1535,9 @@ impl ExecutionPlan for PhraseQueryExec {
                 "column not set for PhraseQuery {}",
                 query.terms
             )))?;
-            let segments = match preset_segments {
-                Some(segments) => segments,
-                None => load_segments(&ds, &column)
-                    .await?
-                    .ok_or(DataFusionError::Execution(format!(
-                        "No Inverted index found for column {}",
-                        column,
-                    )))?,
-            };
+            let segments = segment_selection
+                .resolve(&ds, &column, &metrics.segment_bind_duration)
+                .await?;
             let _details = load_segment_details(&ds, &column, &segments).await?;
             let indices =
                 open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?;
@@ -1395,11 +1567,16 @@ impl ExecutionPlan for PhraseQueryExec {
             let tokens = collect_query_tokens(&query.terms, &mut tokenizer);
             let base_scorer = match preset_base_scorer {
                 Some(scorer) => scorer,
-                None => Arc::new(
-                    build_global_bm25_scorer(&indices, &tokens, &params)
-                        .boxed()
-                        .await?,
-                ),
+                None => {
+                    let scorer_start = std::time::Instant::now();
+                    let scorer = Arc::new(
+                        build_global_bm25_scorer(&indices, &tokens, &params)
+                            .boxed()
+                            .await?,
+                    );
+                    metrics.record_scorer_build(scorer_start.elapsed());
+                    scorer
+                }
             };
 
             pre_filter.wait_for_ready().await?;
@@ -1521,10 +1698,6 @@ impl ExecutionPlan for BoostQueryExec {
         "BoostQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.positive, &self.negative]
     }
@@ -1789,10 +1962,6 @@ impl ExecutionPlan for BooleanQueryExec {
         "BooleanQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.must {
             Some(must) => vec![&self.should, &self.must_not, must],
@@ -1976,10 +2145,11 @@ mod tests {
         UInt64Array,
     };
     use arrow_schema::DataType;
+    use datafusion::error::{DataFusionError, Result as DataFusionResult};
     use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
     use datafusion::{execution::TaskContext, physical_plan::ExecutionPlan};
     use futures::TryStreamExt;
-    use lance_core::ROW_ID;
+    use lance_core::{ROW_ID, utils::address::RowAddress};
     use lance_datafusion::datagen::DatafusionDatagenExt;
     use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts};
     use lance_datafusion::utils::PARTITIONS_SEARCHED_METRIC;
@@ -1995,6 +2165,7 @@ mod tests {
     use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams};
     use lance_index::{IndexCriteria, IndexType};
     use lance_table::format::IndexMetadata;
+    use uuid::Uuid;
 
     use crate::{
         Dataset,
@@ -2006,8 +2177,9 @@ mod tests {
     };
 
     use super::{
-        BoolSlot, BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec,
-        PhraseQueryExec, build_boolean_query_children, open_fts_segments,
+        BoolSlot, BoostQueryExec, FTS_SEGMENT_BIND_DURATION_METRIC, FlatMatchFilterExec,
+        FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec, build_boolean_query_children,
+        open_fts_segments,
     };
     use crate::io::exec::utils::IndexMetrics;
     use datafusion::physical_plan::empty::EmptyExec;
@@ -2033,6 +2205,128 @@ mod tests {
         }
     }
 
+    async fn create_segment_selection_fixture() -> (Arc<Dataset>, Vec<IndexMetadata>, Vec<u32>) {
+        let mut dataset = lance_datagen::gen_batch()
+            .col(
+                "text",
+                lance_datagen::array::cycle_utf8_literals(&["quick brown fox"]),
+            )
+            .col(
+                "other",
+                lance_datagen::array::cycle_utf8_literals(&["not indexed"]),
+            )
+            .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(2))
+            .await
+            .unwrap();
+        let fragment_ids = dataset
+            .get_fragments()
+            .iter()
+            .map(|fragment| fragment.id() as u32)
+            .collect::<Vec<_>>();
+        assert_eq!(fragment_ids.len(), 3);
+
+        let params = InvertedIndexParams::default().with_position(true);
+        let mut segments = Vec::with_capacity(fragment_ids.len());
+        for fragment_id in &fragment_ids {
+            let mut builder = dataset
+                .create_index_builder(&["text"], IndexType::Inverted, &params)
+                .name("segment_selection_fts".to_string())
+                .fragments(vec![*fragment_id]);
+            segments.push(builder.execute_uncommitted().await.unwrap());
+        }
+        dataset
+            .commit_existing_index_segments("segment_selection_fts", "text", segments.clone())
+            .await
+            .unwrap();
+
+        let committed = crate::index::scalar::inverted::load_segments(&dataset, "text")
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(committed.len(), fragment_ids.len());
+        (Arc::new(dataset), committed, fragment_ids)
+    }
+
+    fn segment_uuid_for_fragment(segments: &[IndexMetadata], fragment_id: u32) -> Uuid {
+        segments
+            .iter()
+            .find(|segment| {
+                segment
+                    .fragment_bitmap
+                    .as_ref()
+                    .is_some_and(|fragments| fragments.contains(fragment_id))
+            })
+            .map(|segment| segment.uuid)
+            .unwrap()
+    }
+
+    fn expected_row_ids(fragment_ids: &[u32]) -> Vec<u64> {
+        let mut row_ids = fragment_ids
+            .iter()
+            .flat_map(|fragment_id| {
+                (0..2).map(|offset| u64::from(RowAddress::new_from_parts(*fragment_id, offset)))
+            })
+            .collect::<Vec<_>>();
+        row_ids.sort_unstable();
+        row_ids
+    }
+
+    async fn execute_results(plan: &dyn ExecutionPlan) -> DataFusionResult<Vec<(u64, f32)>> {
+        let batches: Vec<RecordBatch> = plan
+            .execute(0, Arc::new(TaskContext::default()))?
+            .try_collect()
+            .await?;
+        let mut results = Vec::new();
+        for batch in batches {
+            let row_ids = batch[ROW_ID]
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .unwrap();
+            let scores = batch[SCORE_COL]
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap();
+            results.extend(
+                row_ids
+                    .values()
+                    .iter()
+                    .copied()
+                    .zip(scores.values().iter().copied()),
+            );
+        }
+        results.sort_by_key(|(row_id, _)| *row_id);
+        Ok(results)
+    }
+
+    async fn execute_row_ids(plan: &dyn ExecutionPlan) -> DataFusionResult<Vec<u64>> {
+        Ok(execute_results(plan)
+            .await?
+            .into_iter()
+            .map(|(row_id, _)| row_id)
+            .collect())
+    }
+
+    fn metric_value(plan: &dyn ExecutionPlan, name: &str) -> usize {
+        plan.metrics()
+            .unwrap()
+            .iter()
+            .find(|metric| metric.value().name() == name)
+            .unwrap()
+            .value()
+            .as_usize()
+    }
+
+    fn assert_execution_error(error: DataFusionError, expected_message: &str) {
+        assert!(
+            matches!(&error, DataFusionError::Execution(_)),
+            "expected execution error, got {error:?}"
+        );
+        assert!(
+            error.to_string().contains(expected_message),
+            "expected error containing {expected_message:?}, got {error}"
+        );
+    }
+
     #[test]
     fn execute_without_context() {
         // These tests ensure we can create nodes and call execute without a tokio Runtime
@@ -2328,6 +2622,341 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_match_query_exec_segment_selection() {
+        let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await;
+        let query = MatchQuery::new("quick".to_string()).with_column(Some("text".to_string()));
+        let params = FtsSearchParams::default().with_limit(Some(20));
+        let committed_uuids = segments
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+
+        let all_committed = MatchQueryExec::new(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+        );
+        assert!(all_committed.preset_segments().is_none());
+        assert!(all_committed.explicit_segment_uuids().is_none());
+        let all_results = execute_results(&all_committed).await.unwrap();
+        assert_eq!(
+            all_results
+                .iter()
+                .map(|(row_id, _)| *row_id)
+                .collect::<Vec<_>>(),
+            expected_row_ids(&fragment_ids)
+        );
+        assert_eq!(
+            metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0
+        );
+
+        let exact_resolved = MatchQueryExec::new_with_segments(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            segments.clone(),
+        );
+        assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice()));
+        assert_eq!(
+            exact_resolved.explicit_segment_uuids(),
+            Some(committed_uuids.clone())
+        );
+        assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results);
+        assert_eq!(
+            metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0
+        );
+
+        let selected_fragment = fragment_ids[1];
+        let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment);
+        let unpolled = MatchQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        drop(
+            unpolled
+                .execute(0, Arc::new(TaskContext::default()))
+                .unwrap(),
+        );
+        assert_eq!(
+            metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0,
+            "UUID binding should not start until the output stream is polled"
+        );
+
+        let exact_uuids = MatchQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        assert!(exact_uuids.preset_segments().is_none());
+        assert_eq!(
+            exact_uuids.explicit_segment_uuids(),
+            Some(vec![selected_uuid])
+        );
+        assert_eq!(
+            execute_row_ids(&exact_uuids).await.unwrap(),
+            expected_row_ids(&[selected_fragment])
+        );
+        assert!(
+            metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0,
+            "successful UUID binding should record a duration"
+        );
+
+        let input_uuids = vec![
+            segment_uuid_for_fragment(&segments, fragment_ids[2]),
+            segment_uuid_for_fragment(&segments, fragment_ids[0]),
+            segment_uuid_for_fragment(&segments, fragment_ids[2]),
+        ];
+        let deduplicated_uuids = input_uuids[..2].to_vec();
+        let ordered_plan = Arc::new(MatchQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            input_uuids,
+        ))
+        .with_new_children(vec![])
+        .unwrap();
+        let rewritten = ordered_plan.downcast_ref::<MatchQueryExec>().unwrap();
+        assert_eq!(
+            rewritten.explicit_segment_uuids(),
+            Some(deduplicated_uuids.clone())
+        );
+        assert_eq!(
+            execute_row_ids(rewritten).await.unwrap(),
+            expected_row_ids(&[fragment_ids[2], fragment_ids[0]])
+        );
+        let resolver_metrics_set = ExecutionPlanMetricsSet::new();
+        let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0);
+        let resolved = rewritten
+            .segment_selection
+            .resolve(&dataset, "text", &resolver_metrics.segment_bind_duration)
+            .await
+            .unwrap();
+        assert_eq!(
+            resolved
+                .iter()
+                .map(|segment| segment.uuid)
+                .collect::<Vec<_>>(),
+            deduplicated_uuids
+        );
+
+        let empty = MatchQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![],
+        );
+        assert_execution_error(
+            execute_row_ids(&empty).await.unwrap_err(),
+            "requires at least one segment UUID",
+        );
+
+        let missing_uuid = Uuid::new_v4();
+        let missing = MatchQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query,
+            params.clone(),
+            PreFilterSource::None,
+            vec![missing_uuid],
+        );
+        assert_execution_error(
+            execute_row_ids(&missing).await.unwrap_err(),
+            &missing_uuid.to_string(),
+        );
+
+        let wrong_column = MatchQueryExec::new_with_segment_uuids(
+            dataset,
+            MatchQuery::new("quick".to_string()).with_column(Some("other".to_string())),
+            params,
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        assert_execution_error(
+            execute_row_ids(&wrong_column).await.unwrap_err(),
+            "no Inverted index found",
+        );
+    }
+
+    #[tokio::test]
+    async fn test_phrase_query_exec_segment_selection() {
+        let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await;
+        let query =
+            PhraseQuery::new("quick brown".to_string()).with_column(Some("text".to_string()));
+        let params = FtsSearchParams::default().with_limit(Some(20));
+        let committed_uuids = segments
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+
+        let all_committed = PhraseQueryExec::new(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+        );
+        assert!(all_committed.preset_segments().is_none());
+        assert!(all_committed.explicit_segment_uuids().is_none());
+        let all_results = execute_results(&all_committed).await.unwrap();
+        assert_eq!(
+            all_results
+                .iter()
+                .map(|(row_id, _)| *row_id)
+                .collect::<Vec<_>>(),
+            expected_row_ids(&fragment_ids)
+        );
+        assert_eq!(
+            metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0
+        );
+
+        let exact_resolved = PhraseQueryExec::new_with_segments(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            segments.clone(),
+        );
+        assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice()));
+        assert_eq!(
+            exact_resolved.explicit_segment_uuids(),
+            Some(committed_uuids)
+        );
+        assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results);
+        assert_eq!(
+            metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0
+        );
+
+        let selected_fragment = fragment_ids[1];
+        let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment);
+        let unpolled = PhraseQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        drop(
+            unpolled
+                .execute(0, Arc::new(TaskContext::default()))
+                .unwrap(),
+        );
+        assert_eq!(
+            metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC),
+            0,
+            "UUID binding should not start until the output stream is polled"
+        );
+
+        let exact_uuids = PhraseQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        assert!(exact_uuids.preset_segments().is_none());
+        assert_eq!(
+            exact_uuids.explicit_segment_uuids(),
+            Some(vec![selected_uuid])
+        );
+        assert_eq!(
+            execute_row_ids(&exact_uuids).await.unwrap(),
+            expected_row_ids(&[selected_fragment])
+        );
+        assert!(
+            metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0,
+            "successful UUID binding should record a duration"
+        );
+
+        let input_uuids = vec![
+            segment_uuid_for_fragment(&segments, fragment_ids[2]),
+            segment_uuid_for_fragment(&segments, fragment_ids[0]),
+            segment_uuid_for_fragment(&segments, fragment_ids[2]),
+        ];
+        let deduplicated_uuids = input_uuids[..2].to_vec();
+        let ordered_plan = Arc::new(PhraseQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            input_uuids,
+        ))
+        .with_new_children(vec![])
+        .unwrap();
+        let rewritten = ordered_plan.downcast_ref::<PhraseQueryExec>().unwrap();
+        assert_eq!(
+            rewritten.explicit_segment_uuids(),
+            Some(deduplicated_uuids.clone())
+        );
+        assert_eq!(
+            execute_row_ids(rewritten).await.unwrap(),
+            expected_row_ids(&[fragment_ids[2], fragment_ids[0]])
+        );
+        let resolver_metrics_set = ExecutionPlanMetricsSet::new();
+        let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0);
+        let resolved = rewritten
+            .segment_selection
+            .resolve(&dataset, "text", &resolver_metrics.segment_bind_duration)
+            .await
+            .unwrap();
+        assert_eq!(
+            resolved
+                .iter()
+                .map(|segment| segment.uuid)
+                .collect::<Vec<_>>(),
+            deduplicated_uuids
+        );
+
+        let empty = PhraseQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query.clone(),
+            params.clone(),
+            PreFilterSource::None,
+            vec![],
+        );
+        assert_execution_error(
+            execute_row_ids(&empty).await.unwrap_err(),
+            "requires at least one segment UUID",
+        );
+
+        let missing_uuid = Uuid::new_v4();
+        let missing = PhraseQueryExec::new_with_segment_uuids(
+            dataset.clone(),
+            query,
+            params.clone(),
+            PreFilterSource::None,
+            vec![missing_uuid],
+        );
+        assert_execution_error(
+            execute_row_ids(&missing).await.unwrap_err(),
+            &missing_uuid.to_string(),
+        );
+
+        let wrong_column = PhraseQueryExec::new_with_segment_uuids(
+            dataset,
+            PhraseQuery::new("quick brown".to_string()).with_column(Some("other".to_string())),
+            params,
+            PreFilterSource::None,
+            vec![selected_uuid],
+        );
+        assert_execution_error(
+            execute_row_ids(&wrong_column).await.unwrap_err(),
+            "no Inverted index found",
+        );
+    }
+
     #[tokio::test]
     async fn test_match_query_exec_with_base_scorer_matches_baseline() {
         let test_dir = tempfile::tempdir().unwrap();
@@ -2542,7 +3171,7 @@ mod tests {
             .unwrap()
             .expect("Should slot always returns Some");
         assert!(
-            plan.as_any().downcast_ref::<EmptyExec>().is_some(),
+            plan.downcast_ref::<EmptyExec>().is_some(),
             "expected EmptyExec for empty Should slot, got {plan:?}"
         );
     }
@@ -2570,12 +3199,10 @@ mod tests {
         .unwrap()
         .expect("Should slot always returns Some");
         let repartition = plan
-            .as_any()
             .downcast_ref::<RepartitionExec>()
             .expect("multi-child Should should be wrapped in RepartitionExec");
         let inner = repartition
             .input()
-            .as_any()
             .downcast_ref::<UnionExec>()
             .expect("RepartitionExec should wrap a UnionExec");
         assert_eq!(inner.children().len(), 2);
@@ -2614,7 +3241,7 @@ mod tests {
         // there are N-1 joins.
         let mut joins = 0usize;
         let mut current: Arc<dyn ExecutionPlan> = plan;
-        while let Some(join) = current.clone().as_any().downcast_ref::<HashJoinExec>() {
+        while let Some(join) = current.clone().downcast_ref::<HashJoinExec>() {
             joins += 1;
             current = join.children()[0].clone();
         }
@@ -2630,12 +3257,10 @@ mod tests {
         .unwrap()
         .expect("MustNot slot always returns Some");
         let repartition = plan
-            .as_any()
             .downcast_ref::<RepartitionExec>()
             .expect("multi-child MustNot should be wrapped in RepartitionExec");
         let inner = repartition
             .input()
-            .as_any()
             .downcast_ref::<UnionExec>()
             .expect("RepartitionExec should wrap a UnionExec");
         assert_eq!(inner.children().len(), 2);
diff --git a/rust/lance/src/io/exec/knn.rs b/rust/lance/src/io/exec/knn.rs
index f6332ada94e..83aa97b5a6c 100644
--- a/rust/lance/src/io/exec/knn.rs
+++ b/rust/lance/src/io/exec/knn.rs
@@ -3,14 +3,13 @@
 
 #[cfg(test)]
 use lance_core::utils::row_addr_remap::RowAddrRemap;
-use std::any::Any;
 use std::cmp::Ordering as CmpOrdering;
 use std::collections::{BinaryHeap, HashMap, HashSet};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::{Arc, LazyLock, Mutex};
 use std::time::Instant;
 
-use arrow::array::{Float32Builder, Int32Builder};
+use arrow::array::{Float32Builder, Int32Builder, UInt64Builder};
 use arrow::datatypes::{Float32Type, UInt32Type, UInt64Type};
 use arrow_array::{Array, Float32Array, UInt32Array, UInt64Array};
 use arrow_array::{
@@ -61,6 +60,8 @@ use lance_table::format::IndexMetadata;
 use tokio::sync::Notify;
 use uuid::Uuid;
 
+use lance_select::RowAddrMask;
+
 use crate::dataset::Dataset;
 use crate::index::DatasetIndexInternalExt;
 use crate::index::prefilter::{DatasetPreFilter, FilterLoader};
@@ -137,6 +138,37 @@ fn normalize_query_for_index(index: &dyn VectorIndex, query: Query) -> DataFusio
     Ok(query)
 }
 
+/// Normalize a batch query's concatenated key for a cosine index.
+///
+/// `query.key` holds `query_count` vectors of length `dim` concatenated, so each
+/// vector must be normalized **independently** — normalizing the whole buffer
+/// would divide every vector by a single global norm that depends on the other
+/// queries in the batch, corrupting per-query cosine distances. Returns the
+/// query unchanged for non-cosine metrics.
+fn normalize_batch_query_for_index(
+    index: &dyn VectorIndex,
+    mut query: Query,
+    query_count: usize,
+    dim: usize,
+) -> DataFusionResult<Query> {
+    if index.metric_type() != DistanceType::Cosine {
+        return Ok(query);
+    }
+
+    let normalized: Vec<ArrayRef> = (0..query_count)
+        .map(|i| {
+            normalize_arrow(&query.key.slice(i * dim, dim))
+                .map(|(key, _)| key)
+                .map_err(|e| DataFusionError::Execution(format!("Failed to normalize query: {e}")))
+        })
+        .collect::<DataFusionResult<_>>()?;
+    let refs: Vec<&dyn Array> = normalized.iter().map(|a| a.as_ref()).collect();
+    query.key = arrow_select::concat::concat(&refs).map_err(|e| {
+        DataFusionError::Execution(format!("Failed to concat normalized query: {e}"))
+    })?;
+    Ok(query)
+}
+
 /// [ExecutionPlan] compute vector distance from a query vector.
 ///
 /// Preconditions:
@@ -829,10 +861,6 @@ impl ExecutionPlan for KNNVectorDistanceExec {
         "KNNVectorDistanceExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     /// Flat KNN inherits the schema from input node, and add one distance column.
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.output_schema.clone()
@@ -948,7 +976,7 @@ impl ExecutionPlan for KNNVectorDistanceExec {
         Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         let inner_stats = self.input.partition_statistics(partition)?;
         let input_schema = self.input.schema();
         let input_stats_by_name = inner_stats
@@ -985,11 +1013,11 @@ impl ExecutionPlan for KNNVectorDistanceExec {
                 }
             })
             .collect::<Vec<_>>();
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: inner_stats.num_rows,
             column_statistics,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -1104,11 +1132,47 @@ pub static KNN_PARTITION_SCHEMA: LazyLock<SchemaRef> = LazyLock::new(|| {
     ]))
 });
 
+/// Build the shared [`DatasetPreFilter`] for an ANN search node, executing the
+/// prefilter source (if any) for this partition. Used by both the single-query
+/// [`ANNIvfSubIndexExec`] and the batch [`ANNIvfBatchExec`] so the prefilter is
+/// wired identically (and, for a batch, built once and shared across queries).
+///
+/// `overlay_block`, when `Some`, excludes rows whose index entries may be stale
+/// due to a newer data overlay (see [`DatasetPreFilter::with_overlay_block`]).
+fn build_dataset_prefilter(
+    dataset: Arc<Dataset>,
+    indices: &[IndexMetadata],
+    prefilter_source: &PreFilterSource,
+    partition: usize,
+    context: Arc<datafusion::execution::context::TaskContext>,
+    overlay_block: Option<RowAddrMask>,
+) -> DataFusionResult<Arc<DatasetPreFilter>> {
+    let prefilter_loader = match prefilter_source {
+        PreFilterSource::FilteredRowIds(src_node) => {
+            let stream = src_node.execute(partition, context)?;
+            Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box<dyn FilterLoader>)
+        }
+        PreFilterSource::ScalarIndexQuery(src_node) => {
+            let stream = src_node.execute(partition, context)?;
+            Some(Box::new(SelectionVectorToPrefilter(stream)) as Box<dyn FilterLoader>)
+        }
+        PreFilterSource::None => None,
+    };
+    let mut pre_filter = DatasetPreFilter::new(dataset, indices, prefilter_loader);
+    if let Some(overlay_block) = overlay_block {
+        pre_filter = pre_filter.with_overlay_block(overlay_block);
+    }
+    Ok(Arc::new(pre_filter))
+}
+
+/// Create a new ANN execution node. `overlay_block`, when `Some`, excludes rows whose index
+/// entries may be stale due to a newer data overlay (see [`ANNIvfSubIndexExec::with_overlay_block`]).
 pub fn new_knn_exec(
     dataset: Arc<Dataset>,
     indices: &[IndexMetadata],
     query: &Query,
     prefilter_source: PreFilterSource,
+    overlay_block: Option<RowAddrMask>,
 ) -> Result<Arc<dyn ExecutionPlan>> {
     let ivf_node = ANNIvfPartitionExec::try_new(
         dataset.clone(),
@@ -1116,13 +1180,16 @@ pub fn new_knn_exec(
         query.clone(),
     )?;
 
-    let sub_index = ANNIvfSubIndexExec::try_new(
+    let mut sub_index = ANNIvfSubIndexExec::try_new(
         Arc::new(ivf_node),
         dataset,
         indices.to_vec(),
         query.clone(),
         prefilter_source,
     )?;
+    if let Some(overlay_block) = overlay_block {
+        sub_index = sub_index.with_overlay_block(overlay_block);
+    }
 
     Ok(Arc::new(sub_index))
 }
@@ -1225,10 +1292,6 @@ impl ExecutionPlan for ANNIvfPartitionExec {
         "ANNIVFPartitionExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         KNN_PARTITION_SCHEMA.clone()
     }
@@ -1237,11 +1300,11 @@ impl ExecutionPlan for ANNIvfPartitionExec {
         &self.properties
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.minimum_nprobes),
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -1386,6 +1449,10 @@ pub struct ANNIvfSubIndexExec {
     /// Prefiltering input
     prefilter_source: PreFilterSource,
 
+    /// Row addresses whose index entries are stale due to a newer data overlay. Blocked from
+    /// index results at execution time via [`DatasetPreFilter::with_overlay_block`].
+    overlay_block: Option<RowAddrMask>,
+
     /// Datafusion Plan Properties
     properties: Arc<PlanProperties>,
 
@@ -1418,11 +1485,18 @@ impl ANNIvfSubIndexExec {
             indices,
             query,
             prefilter_source,
+            overlay_block: None,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         })
     }
 
+    /// Block stale row addresses (see the `overlay_block` field) from index results.
+    pub fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self {
+        self.overlay_block = Some(overlay_block);
+        self
+    }
+
     /// Returns a reference to the vector query.
     pub fn query(&self) -> &Query {
         &self.query
@@ -1843,10 +1917,6 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
         "ANNSubIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         KNN_INDEX_SCHEMA.clone()
     }
@@ -1893,6 +1963,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                 indices: self.indices.clone(),
                 query: self.query.clone(),
                 prefilter_source,
+                overlay_block: self.overlay_block.clone(),
                 properties: self.properties.clone(),
                 metrics: ExecutionPlanMetricsSet::new(),
             }
@@ -1961,23 +2032,14 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                 async move { DataFusionResult::Ok(stream::iter(plan)) }
             })
             .try_flatten();
-        let prefilter_loader = match &prefilter_source {
-            PreFilterSource::FilteredRowIds(src_node) => {
-                let stream = src_node.execute(partition, context)?;
-                Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box<dyn FilterLoader>)
-            }
-            PreFilterSource::ScalarIndexQuery(src_node) => {
-                let stream = src_node.execute(partition, context)?;
-                Some(Box::new(SelectionVectorToPrefilter(stream)) as Box<dyn FilterLoader>)
-            }
-            PreFilterSource::None => None,
-        };
-
-        let pre_filter = Arc::new(DatasetPreFilter::new(
+        let pre_filter = build_dataset_prefilter(
             ds.clone(),
             &indices,
-            prefilter_loader,
-        ));
+            &prefilter_source,
+            partition,
+            context,
+            self.overlay_block.clone(),
+        )?;
 
         let state = Arc::new(ANNIvfEarlySearchResults::new(indices.len(), query.k));
 
@@ -2043,8 +2105,8 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> DataFusionResult<datafusion::physical_plan::Statistics> {
-        Ok(Statistics {
+    ) -> DataFusionResult<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(
                 self.query.k
                     * self.query.refine_factor.unwrap_or(1) as usize
@@ -2056,7 +2118,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                         .unwrap_or(&1),
             ),
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -2072,6 +2134,327 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
     }
 }
 
+/// Build a batch (multi-query) indexed vector search plan.
+///
+/// `query.key` must hold all `query_count` query vectors concatenated.
+pub fn new_knn_batch_exec(
+    dataset: Arc<Dataset>,
+    indices: &[IndexMetadata],
+    query: &Query,
+    query_count: usize,
+    prefilter_source: PreFilterSource,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    Ok(Arc::new(ANNIvfBatchExec::try_new(
+        dataset,
+        indices.to_vec(),
+        query.clone(),
+        query_count,
+        prefilter_source,
+    )?))
+}
+
+/// [ExecutionPlan] for batch (multi-query) IVF vector search.
+///
+/// Where the single-query path uses [`ANNIvfPartitionExec`] +
+/// [`ANNIvfSubIndexExec`], this node ranks every query vector against the IVF
+/// centroids and then asks the index to read each probed partition's storage
+/// once, scoring all queries that probe it
+/// (via [`VectorIndex::search_partitions_batch`]). The prefilter is built once
+/// and shared across all queries.
+///
+/// This is a separate node rather than a mode on the two single-query nodes
+/// because the two-node pipeline streams one partition-list per delta through a
+/// per-query top-k, whereas the shared scan must invert queries onto partitions
+/// and keep one heap per query in a single pass. It still reuses the underlying
+/// primitives (partition load, prefilter wiring via [`build_dataset_prefilter`],
+/// and the per-partition accumulate the index performs).
+///
+/// Output schema: `{query_index: Int32, _distance: Float32, _rowid: UInt64}`,
+/// sorted by `(query_index, _distance, _rowid)`, with up to `k` rows per query.
+///
+/// Per-query nprobes are honored statically from the ranking; the adaptive
+/// late-search expansion used by the single-query path is not applied, so recall
+/// matches repeated single-query search when `minimum_nprobes == maximum_nprobes`.
+#[derive(Debug)]
+pub struct ANNIvfBatchExec {
+    dataset: Arc<Dataset>,
+    indices: Vec<IndexMetadata>,
+    /// Vector query whose `key` holds all `query_count` vectors concatenated.
+    query: Query,
+    query_count: usize,
+    prefilter_source: PreFilterSource,
+    properties: Arc<PlanProperties>,
+    metrics: ExecutionPlanMetricsSet,
+}
+
+impl ANNIvfBatchExec {
+    pub fn try_new(
+        dataset: Arc<Dataset>,
+        indices: Vec<IndexMetadata>,
+        query: Query,
+        query_count: usize,
+        prefilter_source: PreFilterSource,
+    ) -> Result<Self> {
+        if indices.is_empty() {
+            return Err(Error::index(
+                "ANNIvfBatchExec: no index found for query".to_string(),
+            ));
+        }
+        if query_count == 0 || !query.key.len().is_multiple_of(query_count) {
+            return Err(Error::invalid_input(format!(
+                "ANNIvfBatchExec: query key length {} is not divisible by query count {query_count}",
+                query.key.len()
+            )));
+        }
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(knn_empty_result_schema(true)),
+            Partitioning::RoundRobinBatch(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        Ok(Self {
+            dataset,
+            indices,
+            query,
+            query_count,
+            prefilter_source,
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+        })
+    }
+}
+
+impl DisplayAs for ANNIvfBatchExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                write!(
+                    f,
+                    "ANNIvfBatch: query_count={}, k={}, deltas={}",
+                    self.query_count,
+                    self.query.k,
+                    self.indices.len()
+                )
+            }
+            DisplayFormatType::TreeRender => {
+                write!(
+                    f,
+                    "ANNIvfBatch\nquery_count={}\nk={}\ndeltas={}",
+                    self.query_count,
+                    self.query.k,
+                    self.indices.len()
+                )
+            }
+        }
+    }
+}
+
+impl ExecutionPlan for ANNIvfBatchExec {
+    fn name(&self) -> &str {
+        "ANNIvfBatchExec"
+    }
+
+    fn schema(&self) -> SchemaRef {
+        knn_empty_result_schema(true)
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.properties
+    }
+
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        match &self.prefilter_source {
+            PreFilterSource::None => vec![],
+            PreFilterSource::FilteredRowIds(src) => vec![src],
+            PreFilterSource::ScalarIndexQuery(src) => vec![src],
+        }
+    }
+
+    fn required_input_distribution(&self) -> Vec<Distribution> {
+        self.children()
+            .iter()
+            .map(|_| Distribution::SinglePartition)
+            .collect()
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        mut children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> DataFusionResult<Arc<dyn ExecutionPlan>> {
+        let prefilter_source = match (&self.prefilter_source, children.len()) {
+            (PreFilterSource::None, 0) => PreFilterSource::None,
+            (PreFilterSource::FilteredRowIds(_), 1) => {
+                PreFilterSource::FilteredRowIds(children.pop().expect("length checked"))
+            }
+            (PreFilterSource::ScalarIndexQuery(_), 1) => {
+                PreFilterSource::ScalarIndexQuery(children.pop().expect("length checked"))
+            }
+            _ => {
+                return Err(DataFusionError::Internal(
+                    "ANNIvfBatchExec given an unexpected number of children".to_string(),
+                ));
+            }
+        };
+        Ok(Arc::new(Self {
+            dataset: self.dataset.clone(),
+            indices: self.indices.clone(),
+            query: self.query.clone(),
+            query_count: self.query_count,
+            prefilter_source,
+            properties: self.properties.clone(),
+            metrics: ExecutionPlanMetricsSet::new(),
+        }))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<datafusion::execution::context::TaskContext>,
+    ) -> DataFusionResult<SendableRecordBatchStream> {
+        let schema = self.schema();
+        let ds = self.dataset.clone();
+        let column = self.query.column.clone();
+        let indices = self.indices.clone();
+        let query = self.query.clone();
+        let query_count = self.query_count;
+        let metrics = Arc::new(AnnIndexMetrics::new(&self.metrics, partition));
+        let metrics_clone = metrics.clone();
+        let timer = Instant::now();
+
+        let pre_filter = build_dataset_prefilter(
+            ds.clone(),
+            &indices,
+            &self.prefilter_source,
+            partition,
+            context,
+            // The batch node has no data overlay to reconcile against, so no
+            // stale-row block is applied (see `ANNIvfSubIndexExec::overlay_block`).
+            None,
+        )?;
+
+        let result_schema = schema.clone();
+        let fut = async move {
+            let dim = query.key.len() / query_count;
+            // Per-query candidate (distance, row_id) pairs accumulated across deltas.
+            let mut candidates: Vec<Vec<(f32, u64)>> = vec![Vec::new(); query_count];
+
+            for index_meta in &indices {
+                let index = ds
+                    .open_vector_index(&column, &index_meta.uuid, &metrics.index_metrics)
+                    .await?;
+                // The scanner's `batch_index_search_supported` gate decides which
+                // indices reach this node; this check only guards against that
+                // gate and the index implementation disagreeing (an internal
+                // invariant, not a user-facing error).
+                if !index.supports_batch_partition_search() {
+                    return Err(DataFusionError::Internal(format!(
+                        "ANNIvfBatchExec reached for index {} that does not support batch \
+                         partition search",
+                        index_meta.uuid
+                    )));
+                }
+                // Normalize each query vector independently (cosine only)
+                // before ranking; see normalize_batch_query_for_index.
+                let normalized = normalize_batch_query_for_index(
+                    index.as_ref(),
+                    query.clone(),
+                    query_count,
+                    dim,
+                )?;
+
+                let mut partitions_per_query = Vec::with_capacity(query_count);
+                let mut dists_per_query = Vec::with_capacity(query_count);
+                for query_index in 0..query_count {
+                    let mut single_query = normalized.clone();
+                    single_query.key = normalized.key.slice(query_index * dim, dim);
+                    // Probe a fixed number of partitions per query. The scanner
+                    // only routes here when `minimum_nprobes == maximum_nprobes`
+                    // (see `Scanner::batch_index_search_supported`), so this is
+                    // exactly what the single-query path would search — no
+                    // adaptive `early_pruning` floor or late-search expansion
+                    // applies, making the batch result identical to repeated
+                    // single-query search.
+                    let nprobes = single_query.minimum_nprobes.max(1);
+                    single_query.maximum_nprobes = Some(nprobes);
+                    let (partitions, q_c_dists) = index.find_partitions(&single_query)?;
+                    partitions_per_query.push(Arc::new(partitions));
+                    dists_per_query.push(Arc::new(q_c_dists));
+                }
+
+                let index_metrics: Arc<dyn MetricsCollector> =
+                    Arc::new(metrics.index_metrics.clone());
+                let pre_filter: Arc<dyn PreFilter> = pre_filter.clone();
+                let per_query = index
+                    .search_partitions_batch(
+                        normalized,
+                        partitions_per_query,
+                        dists_per_query,
+                        pre_filter,
+                        index_metrics,
+                    )
+                    .await?;
+
+                for (query_index, batch) in per_query.into_iter().enumerate() {
+                    let dists = batch.column(0).as_primitive::<Float32Type>();
+                    let row_ids = batch.column(1).as_primitive::<UInt64Type>();
+                    candidates[query_index].extend(
+                        dists
+                            .values()
+                            .iter()
+                            .copied()
+                            .zip(row_ids.values().iter().copied()),
+                    );
+                }
+            }
+
+            // Per-query top-k merge across deltas, tagged with query_index.
+            let mut query_index_builder = Int32Builder::new();
+            let mut distance_builder = Float32Builder::new();
+            let mut row_id_builder = UInt64Builder::new();
+            for (query_index, cands) in candidates.iter_mut().enumerate() {
+                cands.sort_by(|a, b| a.0.total_cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
+                cands.truncate(query.k);
+                for (distance, row_id) in cands.iter() {
+                    query_index_builder.append_value(query_index as i32);
+                    distance_builder.append_value(*distance);
+                    row_id_builder.append_value(*row_id);
+                }
+            }
+            let batch = RecordBatch::try_new(
+                result_schema,
+                vec![
+                    Arc::new(query_index_builder.finish()),
+                    Arc::new(distance_builder.finish()),
+                    Arc::new(row_id_builder.finish()),
+                ],
+            )?;
+            metrics.baseline_metrics.record_output(batch.num_rows());
+            DataFusionResult::Ok(batch)
+        };
+
+        let stream = stream::once(fut).finally(move || {
+            metrics_clone.index_metrics.flush_io();
+            metrics_clone
+                .baseline_metrics
+                .elapsed_compute()
+                .add_duration(timer.elapsed());
+            metrics_clone.baseline_metrics.done();
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            stream.boxed(),
+        )))
+    }
+
+    fn supports_limit_pushdown(&self) -> bool {
+        false
+    }
+}
+
 fn adjust_probes(query: &mut Query, pruned_nprobes: usize) {
     query.minimum_nprobes = query.minimum_nprobes.max(pruned_nprobes);
     if let Some(maximum) = query.maximum_nprobes
@@ -2139,10 +2522,6 @@ impl ExecutionPlan for MultivectorScoringExec {
         "MultivectorScoringExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         KNN_INDEX_SCHEMA.clone()
     }
@@ -2290,6 +2669,8 @@ impl ExecutionPlan for MultivectorScoringExec {
 mod tests {
     use super::*;
 
+    use std::any::Any;
+
     use crate::index::DatasetIndexExt;
     use arrow::compute::{concat_batches, sort_to_indices, take_record_batch};
     use arrow::datatypes::Float32Type;
@@ -2321,6 +2702,7 @@ mod tests {
 
     use crate::dataset::{WriteMode, WriteParams};
     use crate::index::vector::VectorIndexParams;
+    use crate::index::vector::ivf::v2::STREAMING_SEARCH_BATCH_SIZE;
     use crate::io::exec::testing::TestingExec;
 
     fn base_query() -> Query {
@@ -2634,7 +3016,6 @@ mod tests {
             _metrics: Arc<dyn lance_index::metrics::MetricsCollector>,
         ) -> Result<SendableRecordBatchStream> {
             let (batch_tx, batch_rx) = mpsc::channel(1);
-            let batch_tx_for_search = batch_tx.clone();
             let prepared_partition_ids = (start_idx..end_idx)
                 .map(|idx| partitions.value(idx) as usize)
                 .collect::<Vec<_>>();
@@ -2642,42 +3023,74 @@ mod tests {
                 .lock()
                 .unwrap()
                 .extend(prepared_partition_ids.iter().copied());
+            // Mirror the production streaming path (v2.rs): search prepared partitions
+            // in batches of STREAMING_SEARCH_BATCH_SIZE, one `spawn_cpu` per batch, with
+            // the channel send in async code so no CPU-pool thread parks (#7642).
             tokio::spawn(async move {
-                let search_result = spawn_cpu(move || -> DataFusionResult<()> {
-                    for partition_id in prepared_partition_ids {
-                        if control
-                            .as_ref()
-                            .is_some_and(|control| control.should_stop())
-                        {
-                            return Ok(());
-                        }
-                        let batch = self
-                            .search_prepared_partition(
-                                Box::new(partition_id),
-                                &lance_index::metrics::NoOpMetricsCollector,
-                            )
-                            .map_err(datafusion::error::DataFusionError::from);
-                        match batch {
-                            Ok(batch) => {
-                                if let Some(control) = control.as_ref() {
-                                    control.record_batch(&batch);
+                for chunk in prepared_partition_ids.chunks(*STREAMING_SEARCH_BATCH_SIZE) {
+                    if control
+                        .as_ref()
+                        .is_some_and(|control| control.should_stop())
+                        || batch_tx.is_closed()
+                    {
+                        return;
+                    }
+                    let chunk = chunk.to_vec();
+                    let index = self.clone();
+                    let control_for_search = control.clone();
+                    let cancel_probe = batch_tx.clone();
+                    let search_output = spawn_cpu(move || {
+                        let mut outputs: Vec<DataFusionResult<RecordBatch>> =
+                            Vec::with_capacity(chunk.len());
+                        let mut stopped = false;
+                        for partition_id in chunk {
+                            if control_for_search
+                                .as_ref()
+                                .is_some_and(|control| control.should_stop())
+                                || cancel_probe.is_closed()
+                            {
+                                stopped = true;
+                                break;
+                            }
+                            match index
+                                .search_prepared_partition(
+                                    Box::new(partition_id),
+                                    &lance_index::metrics::NoOpMetricsCollector,
+                                )
+                                .map_err(datafusion::error::DataFusionError::from)
+                            {
+                                Ok(batch) => {
+                                    if let Some(control) = control_for_search.as_ref() {
+                                        control.record_batch(&batch);
+                                    }
+                                    outputs.push(Ok(batch));
                                 }
-                                if batch_tx_for_search.blocking_send(Ok(batch)).is_err() {
-                                    return Ok(());
+                                Err(err) => {
+                                    outputs.push(Err(err));
+                                    stopped = true;
+                                    break;
                                 }
                             }
-                            Err(err) => {
-                                let _ = batch_tx_for_search.blocking_send(Err(err));
-                                return Ok(());
-                            }
                         }
-                    }
-                    Ok(())
-                })
-                .await;
+                        Ok::<_, datafusion::error::DataFusionError>((outputs, stopped))
+                    })
+                    .await;
 
-                if let Err(err) = search_result {
-                    let _ = batch_tx.send(Err(err)).await;
+                    let (outputs, stopped) = match search_output {
+                        Ok(output) => output,
+                        Err(err) => {
+                            let _ = batch_tx.send(Err(err)).await;
+                            return;
+                        }
+                    };
+                    for output in outputs {
+                        if batch_tx.send(output).await.is_err() {
+                            return;
+                        }
+                    }
+                    if stopped {
+                        return;
+                    }
                 }
             });
 
@@ -2871,19 +3284,29 @@ mod tests {
         );
     }
 
+    // All partitions fit in a single search batch, so they are searched in one
+    // `spawn_cpu` dispatch and therefore share one cpu thread. The partition count
+    // adapts to the configured batch size so the single-batch property holds under
+    // any valid `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE`, including 1.
     #[tokio::test]
     async fn test_sequential_initial_search_prepares_all_then_searches_on_one_cpu_thread() {
+        let num_partitions = 3.min(*STREAMING_SEARCH_BATCH_SIZE);
+        let row_ids = (0..num_partitions).map(|i| 10 + i as u64).collect();
         let (index, prepared_partitions, searched_partitions, search_threads) =
-            prepared_index(vec![10, 11, 12]);
+            prepared_index(row_ids);
         let mut query = base_query();
-        query.minimum_nprobes = 3;
+        query.minimum_nprobes = num_partitions;
         let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k));
 
+        let partition_idx = (0..num_partitions as u32).collect::<Vec<_>>();
+        let q_c_dists = (0..num_partitions)
+            .map(|i| i as f32 * 0.1)
+            .collect::<Vec<_>>();
         let batches = ANNIvfSubIndexExec::initial_search(
             index,
             query,
-            Arc::new(UInt32Array::from(vec![0, 1, 2])),
-            Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3])),
+            Arc::new(UInt32Array::from(partition_idx)),
+            Arc::new(Float32Array::from(q_c_dists)),
             empty_prefilter().await,
             prepared_metrics(),
             state,
@@ -2893,11 +3316,12 @@ mod tests {
         .await
         .unwrap();
 
-        assert_eq!(batches.len(), 3);
-        assert_eq!(*prepared_partitions.lock().unwrap(), vec![0, 1, 2]);
-        assert_eq!(*searched_partitions.lock().unwrap(), vec![0, 1, 2]);
+        let expected: Vec<usize> = (0..num_partitions).collect();
+        assert_eq!(batches.len(), num_partitions);
+        assert_eq!(*prepared_partitions.lock().unwrap(), expected);
+        assert_eq!(*searched_partitions.lock().unwrap(), expected);
         let search_threads = search_threads.lock().unwrap().clone();
-        assert_eq!(search_threads.len(), 3);
+        assert_eq!(search_threads.len(), num_partitions);
         assert!(
             search_threads.iter().all(|name| name.contains("lance-cpu")),
             "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}",
@@ -2908,6 +3332,52 @@ mod tests {
         );
     }
 
+    // Regression guard for the batched streaming search (#7642): with more partitions
+    // than a single batch, the search spans multiple `spawn_cpu` dispatches. Verify that
+    // every partition is still prepared and searched in order across the batch boundary,
+    // and that all search work stays on the cpu runtime.
+    //
+    // Note: this does not reproduce the single-thread-pool deadlock the async recv/send
+    // fixes -- that requires a 1-thread CPU pool, which is a process-global singleton and
+    // impractical to force in a unit test (same limitation noted for the #7423 fix).
+    #[tokio::test]
+    async fn test_sequential_search_spans_multiple_cpu_batches() {
+        let num_partitions = *STREAMING_SEARCH_BATCH_SIZE + 3;
+        let row_ids = (0..num_partitions).map(|i| i as u64 * 10).collect();
+        let (index, prepared_partitions, searched_partitions, search_threads) =
+            prepared_index(row_ids);
+        let mut query = base_query();
+        query.minimum_nprobes = num_partitions;
+        let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k));
+
+        let partition_idx = (0..num_partitions as u32).collect::<Vec<_>>();
+        let q_c_dists = (0..num_partitions).map(|i| i as f32).collect::<Vec<_>>();
+        let batches = ANNIvfSubIndexExec::initial_search(
+            index,
+            query,
+            Arc::new(UInt32Array::from(partition_idx.clone())),
+            Arc::new(Float32Array::from(q_c_dists)),
+            empty_prefilter().await,
+            prepared_metrics(),
+            state,
+            usize::MAX,
+        )
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+
+        let expected: Vec<usize> = (0..num_partitions).collect();
+        assert_eq!(batches.len(), num_partitions);
+        assert_eq!(*prepared_partitions.lock().unwrap(), expected);
+        assert_eq!(*searched_partitions.lock().unwrap(), expected);
+        let search_threads = search_threads.lock().unwrap().clone();
+        assert_eq!(search_threads.len(), num_partitions);
+        assert!(
+            search_threads.iter().all(|name| name.contains("lance-cpu")),
+            "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}",
+        );
+    }
+
     #[tokio::test]
     async fn test_sequential_late_search_prepares_all_then_stops_search_early() {
         let (index, prepared_partitions, searched_partitions, _search_threads) =
diff --git a/rust/lance/src/io/exec/optimizer.rs b/rust/lance/src/io/exec/optimizer.rs
index 72488f3a14e..528cc2fa709 100644
--- a/rust/lance/src/io/exec/optimizer.rs
+++ b/rust/lance/src/io/exec/optimizer.rs
@@ -6,6 +6,7 @@
 use std::sync::Arc;
 
 use super::TakeExec;
+use super::filtered_read::FilteredReadExec;
 use arrow_schema::Schema as ArrowSchema;
 #[allow(deprecated)]
 use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
@@ -18,17 +19,38 @@ use datafusion::{
 };
 use datafusion_physical_expr::{PhysicalExpr, expressions::Column};
 
-/// Rule that eliminates [TakeExec] nodes that are immediately followed by another [TakeExec].
+/// Rule that eliminates take nodes that are immediately followed by another
+/// take node, fetching the union of the columns in a single node instead.
+///
+/// A "take" is either a [TakeExec] (legacy storage) or a [FilteredReadExec]
+/// with a row-stream source (see `FilteredReadExec::row_stream_input`); the
+/// scanner emits stacked takes in some plan shapes (e.g. filter columns then
+/// projection columns).
 #[derive(Debug)]
 pub struct CoalesceTake;
 
 impl CoalesceTake {
+    /// Whether `plan` is a take node this rule knows how to collapse
+    fn as_take(plan: &Arc<dyn ExecutionPlan>) -> Option<&dyn ExecutionPlan> {
+        if plan.downcast_ref::<TakeExec>().is_some() {
+            Some(plan.as_ref())
+        } else if let Some(filtered_read) = plan.downcast_ref::<FilteredReadExec>() {
+            filtered_read
+                .row_stream_input()
+                .is_some()
+                .then_some(plan.as_ref())
+        } else {
+            None
+        }
+    }
+
     fn field_order_differs(old_schema: &ArrowSchema, new_schema: &ArrowSchema) -> bool {
-        old_schema
-            .fields
-            .iter()
-            .zip(&new_schema.fields)
-            .any(|(old, new)| old.name() != new.name())
+        old_schema.fields.len() != new_schema.fields.len()
+            || old_schema
+                .fields
+                .iter()
+                .zip(&new_schema.fields)
+                .any(|(old, new)| old.name() != new.name())
     }
 
     fn remap_collapsed_output(
@@ -47,24 +69,39 @@ impl CoalesceTake {
         Arc::new(ProjectionExec::try_new(project_exprs, plan).unwrap())
     }
 
+    /// Collapse two stacked takes into one, or return None when the rebuilt
+    /// node would not produce every column of the original output (the
+    /// rebuild re-derives what to fetch from the outer take's projection, so
+    /// a column only the inner take fetched can go missing if the outer
+    /// projection doesn't cover it)
     fn collapse_takes(
-        inner_take: &TakeExec,
-        outer_take: &TakeExec,
+        inner_take: &dyn ExecutionPlan,
+        outer_take: &dyn ExecutionPlan,
         outer_exec: Arc<dyn ExecutionPlan>,
-    ) -> Arc<dyn ExecutionPlan> {
+    ) -> Option<Arc<dyn ExecutionPlan>> {
         let inner_take_input = inner_take.children()[0].clone();
         let old_output_schema = outer_take.schema();
-        let collapsed = outer_exec
-            .with_new_children(vec![inner_take_input])
-            .unwrap();
+        let collapsed = outer_exec.with_new_children(vec![inner_take_input]).ok()?;
         let new_output_schema = collapsed.schema();
 
+        if old_output_schema
+            .fields()
+            .iter()
+            .any(|field| new_output_schema.field_with_name(field.name()).is_err())
+        {
+            return None;
+        }
+
         // It's possible that collapsing the take can change the field order.  This disturbs DF's planner and
         // so we must restore it.
         if Self::field_order_differs(&old_output_schema, &new_output_schema) {
-            Self::remap_collapsed_output(&old_output_schema, &new_output_schema, collapsed)
+            Some(Self::remap_collapsed_output(
+                &old_output_schema,
+                &new_output_schema,
+                collapsed,
+            ))
         } else {
-            collapsed
+            Some(collapsed)
         }
     }
 }
@@ -78,26 +115,23 @@ impl PhysicalOptimizerRule for CoalesceTake {
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
         Ok(plan
             .transform_down(|plan| {
-                if let Some(outer_take) = plan.as_any().downcast_ref::<TakeExec>() {
+                if let Some(outer_take) = Self::as_take(&plan) {
                     let child = outer_take.children()[0];
-                    // Case 1: TakeExec -> TakeExec
-                    if let Some(inner_take) = child.as_any().downcast_ref::<TakeExec>() {
-                        return Ok(Transformed::yes(Self::collapse_takes(
-                            inner_take,
-                            outer_take,
-                            plan.clone(),
-                        )));
-                    // Case 2: TakeExec -> CoalesceBatchesExec -> TakeExec
-                    } else if let Some(exec_child) =
-                        child.as_any().downcast_ref::<CoalesceBatchesExec>()
-                    {
+                    // Case 1: take -> take
+                    if let Some(inner_take) = Self::as_take(child) {
+                        if let Some(collapsed) =
+                            Self::collapse_takes(inner_take, outer_take, plan.clone())
+                        {
+                            return Ok(Transformed::yes(collapsed));
+                        }
+                    // Case 2: take -> CoalesceBatchesExec -> take
+                    } else if let Some(exec_child) = child.downcast_ref::<CoalesceBatchesExec>() {
                         let inner_child = exec_child.children()[0].clone();
-                        if let Some(inner_take) = inner_child.as_any().downcast_ref::<TakeExec>() {
-                            return Ok(Transformed::yes(Self::collapse_takes(
-                                inner_take,
-                                outer_take,
-                                plan.clone(),
-                            )));
+                        if let Some(inner_take) = Self::as_take(&inner_child)
+                            && let Some(collapsed) =
+                                Self::collapse_takes(inner_take, outer_take, plan.clone())
+                        {
+                            return Ok(Transformed::yes(collapsed));
                         }
                     }
                 }
@@ -128,7 +162,7 @@ impl PhysicalOptimizerRule for SimplifyProjection {
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
         Ok(plan
             .transform_down(|plan| {
-                if let Some(proj) = plan.as_any().downcast_ref::<ProjectionExec>() {
+                if let Some(proj) = plan.downcast_ref::<ProjectionExec>() {
                     let children = proj.children();
                     if children.len() != 1 {
                         return Ok(Transformed::no(plan));
@@ -145,7 +179,7 @@ impl PhysicalOptimizerRule for SimplifyProjection {
                     }
 
                     if proj.expr().iter().enumerate().all(|(index, proj_expr)| {
-                        if let Some(expr) = proj_expr.expr.as_any().downcast_ref::<Column>() {
+                        if let Some(expr) = proj_expr.expr.downcast_ref::<Column>() {
                             // no renaming, no reordering
                             expr.index() == index && expr.name() == proj_expr.alias
                         } else {
@@ -184,3 +218,196 @@ pub fn get_physical_optimizer() -> PhysicalOptimizer {
         Arc::new(datafusion::physical_optimizer::enforce_distribution::EnforceDistribution::new()),
     ])
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use arrow_array::cast::AsArray;
+    use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt64Array};
+    use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+    use arrow_select::concat::concat_batches;
+    use datafusion::execution::TaskContext;
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+    use futures::TryStreamExt;
+    use lance_core::ROW_ID;
+    use lance_core::datatypes::OnMissing;
+    use lance_core::utils::tempfile::TempStrDir;
+    use lance_datafusion::exec::OneShotExec;
+    use lance_file::version::LanceFileVersion;
+
+    use crate::dataset::{Dataset, WriteParams};
+    use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions};
+
+    /// 20 rows, one fragment, columns i (Int32) and s (Utf8)
+    async fn fixture(storage_version: LanceFileVersion) -> (Arc<Dataset>, TempStrDir) {
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, false),
+            ArrowField::new("s", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..20)),
+                Arc::new(StringArray::from_iter_values(
+                    (0..20).map(|v| format!("s-{v}")),
+                )),
+            ],
+        )
+        .unwrap();
+        let tmp_dir = TempStrDir::default();
+        let uri = tmp_dir.as_str();
+        let params = WriteParams {
+            data_storage_version: Some(storage_version),
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        Dataset::write(reader, uri, Some(params)).await.unwrap();
+        (Arc::new(Dataset::open(uri).await.unwrap()), tmp_dir)
+    }
+
+    /// An input plan producing one batch of `_rowid` keys
+    fn keys_input(keys: Vec<u64>) -> Arc<dyn ExecutionPlan> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            ROW_ID,
+            DataType::UInt64,
+            true,
+        )]));
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(UInt64Array::from(keys))]).unwrap();
+        let stream = futures::stream::iter(vec![Ok(batch)]);
+        let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+        Arc::new(OneShotExec::new(stream))
+    }
+
+    fn row_stream_take(
+        dataset: &Arc<Dataset>,
+        input: Arc<dyn ExecutionPlan>,
+        columns: &[&str],
+    ) -> Arc<dyn ExecutionPlan> {
+        // Mirror Scanner::take: full target projection, carried identity kept
+        let mut projection = dataset
+            .empty_projection()
+            .union_columns(columns, OnMissing::Error)
+            .unwrap();
+        projection.with_row_id = true;
+        Arc::new(
+            FilteredReadExec::try_new(
+                dataset.clone(),
+                FilteredReadOptions::new(projection),
+                Some(input),
+            )
+            .unwrap(),
+        )
+    }
+
+    async fn run(plan: &Arc<dyn ExecutionPlan>) -> RecordBatch {
+        let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+        let schema = stream.schema();
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        concat_batches(&schema, batches.iter()).unwrap()
+    }
+
+    fn count_takes(plan: &Arc<dyn ExecutionPlan>) -> usize {
+        let self_count = CoalesceTake::as_take(plan).map(|_| 1).unwrap_or(0);
+        self_count
+            + plan
+                .children()
+                .iter()
+                .map(|child| count_takes(child))
+                .sum::<usize>()
+    }
+
+    /// Stacked row-stream takes collapse into one node fetching both takes'
+    /// columns, preserving the output schema and values
+    #[tokio::test]
+    async fn collapse_row_stream_takes() {
+        let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await;
+
+        // OneShotExec inputs are single-use: build the plan fresh per run
+        let build = |dataset: &Arc<Dataset>| {
+            let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]);
+            row_stream_take(dataset, inner, &["i", "s"])
+        };
+        let outer = build(&dataset);
+        let expected_schema = outer.schema();
+        assert_eq!(count_takes(&outer), 2);
+        let expected = run(&outer).await;
+
+        let optimized = CoalesceTake
+            .optimize(build(&dataset), &ConfigOptions::default())
+            .unwrap();
+        assert_eq!(count_takes(&optimized), 1);
+        assert_eq!(optimized.schema(), expected_schema);
+
+        let result = run(&optimized).await;
+        assert_eq!(result, expected);
+        let i_col = result
+            .column_by_name("i")
+            .unwrap()
+            .as_primitive::<arrow::datatypes::Int32Type>();
+        assert_eq!(i_col.values(), &[3, 1, 4]);
+    }
+
+    /// When the outer take's projection does not cover a column the inner
+    /// take fetched, the collapse is skipped instead of dropping the column
+    #[tokio::test]
+    async fn collapse_skipped_when_column_would_drop() {
+        let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await;
+
+        // Outer target deliberately omits the inner take's "s"
+        let build = |dataset: &Arc<Dataset>| {
+            let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]);
+            row_stream_take(dataset, inner, &["i"])
+        };
+        let outer = build(&dataset);
+        assert_eq!(count_takes(&outer), 2);
+        let expected = run(&outer).await;
+
+        let optimized = CoalesceTake
+            .optimize(build(&dataset), &ConfigOptions::default())
+            .unwrap();
+        assert_eq!(count_takes(&optimized), 2);
+        assert_eq!(run(&optimized).await, expected);
+        assert!(expected.column_by_name("s").is_some());
+    }
+
+    /// Legacy TakeExec pairs still collapse (through the CoalesceBatchesExec
+    /// the scanner inserts on that path)
+    #[tokio::test]
+    async fn collapse_legacy_takes() {
+        let (dataset, _tmp) = fixture(LanceFileVersion::Legacy).await;
+
+        let build = |dataset: &Arc<Dataset>| -> Arc<dyn ExecutionPlan> {
+            let inner_proj = dataset
+                .empty_projection()
+                .union_columns(["s"], OnMissing::Error)
+                .unwrap();
+            let inner: Arc<dyn ExecutionPlan> = Arc::new(
+                TakeExec::try_new(dataset.clone(), keys_input(vec![3, 1, 4]), inner_proj)
+                    .unwrap()
+                    .unwrap(),
+            );
+            let outer_proj = dataset
+                .empty_projection()
+                .union_columns(["i", "s"], OnMissing::Error)
+                .unwrap();
+            Arc::new(
+                TakeExec::try_new(dataset.clone(), inner, outer_proj)
+                    .unwrap()
+                    .unwrap(),
+            )
+        };
+        let outer = build(&dataset);
+        let expected_schema = outer.schema();
+        assert_eq!(count_takes(&outer), 2);
+        let expected = run(&outer).await;
+
+        let optimized = CoalesceTake
+            .optimize(build(&dataset), &ConfigOptions::default())
+            .unwrap();
+        assert_eq!(count_takes(&optimized), 1);
+        assert_eq!(optimized.schema(), expected_schema);
+        assert_eq!(run(&optimized).await, expected);
+    }
+}
diff --git a/rust/lance/src/io/exec/projection.rs b/rust/lance/src/io/exec/projection.rs
index 3106fcfac61..06bc0b8de67 100644
--- a/rust/lance/src/io/exec/projection.rs
+++ b/rust/lance/src/io/exec/projection.rs
@@ -44,7 +44,7 @@ pub fn project(input: Arc<dyn ExecutionPlan>, projection: &ArrowSchema) -> Resul
 
     let field_names = projection.fields().iter().map(|f| f.name()).cloned();
 
-    for (name, selection) in field_names.zip(selections.into_iter()) {
+    for (name, selection) in field_names.zip(selections) {
         let expr = selection_as_expr(&selection, input_schema.fields(), None);
         exprs.push((expr, name));
     }
diff --git a/rust/lance/src/io/exec/pushdown_scan.rs b/rust/lance/src/io/exec/pushdown_scan.rs
index b0b0eacded6..b82434116b8 100644
--- a/rust/lance/src/io/exec/pushdown_scan.rs
+++ b/rust/lance/src/io/exec/pushdown_scan.rs
@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use std::collections::HashMap;
-use std::{any::Any, sync::Arc};
+use std::sync::Arc;
 
 use arrow_array::cast::AsArray;
 use arrow_array::types::{Int64Type, UInt64Type};
@@ -25,7 +25,8 @@ use datafusion::{
 };
 use datafusion_functions::core::expr_ext::FieldAccessor;
 use datafusion_physical_expr::EquivalenceProperties;
-use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
+use futures::stream::BoxStream;
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use lance_arrow::{RecordBatchExt, SchemaExt};
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::{ROW_ADDR, ROW_ADDR_FIELD, ROW_ID_FIELD};
@@ -158,10 +159,6 @@ impl ExecutionPlan for LancePushdownScanExec {
         "LancePushdownScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -325,7 +322,7 @@ impl FragmentScanner {
         })
     }
 
-    pub fn scan(self) -> Result<impl Stream<Item = Result<RecordBatch>> + 'static + Send> {
+    pub fn scan(self) -> Result<BoxStream<'static, Result<RecordBatch>>> {
         let batch_readahead = self.config.batch_readahead;
         let simplified_predicates = self.simplified_predicates()?;
         let ordered_output = self.config.ordered_output;
@@ -671,7 +668,7 @@ impl FragmentScanner {
                 .collect();
             let schema =
                 Arc::new(ArrowSchema::from(self.predicate_projection.as_ref()).try_into()?);
-            let context = SimplifyContext::default().with_schema(schema);
+            let context = SimplifyContext::builder().with_schema(schema).build();
             let mut simplifier = ExprSimplifier::new(context);
 
             let mut predicates = Vec::with_capacity(num_batches);
diff --git a/rust/lance/src/io/exec/rowids.rs b/rust/lance/src/io/exec/rowids.rs
index 837d0b81fa3..dc9a54cc182 100644
--- a/rust/lance/src/io/exec/rowids.rs
+++ b/rust/lance/src/io/exec/rowids.rs
@@ -242,10 +242,6 @@ impl ExecutionPlan for AddRowAddrExec {
         "AddRowAddrExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> Arc<Schema> {
         self.output_schema.clone()
     }
@@ -291,8 +287,8 @@ impl ExecutionPlan for AddRowAddrExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> Result<datafusion::physical_plan::Statistics> {
-        let mut stats = self.input.partition_statistics(partition)?;
+    ) -> Result<Arc<datafusion::physical_plan::Statistics>> {
+        let mut stats = Arc::unwrap_or_clone(self.input.partition_statistics(partition)?);
 
         let row_id_col_stats = stats.column_statistics.get(self.rowid_pos).ok_or_else(|| {
             DataFusionError::Internal("RowAddrExec: rowid column stats not found".into())
@@ -327,7 +323,7 @@ impl ExecutionPlan for AddRowAddrExec {
             .column_statistics
             .insert(self.rowaddr_pos, row_addr_col_stats);
 
-        Ok(stats)
+        Ok(Arc::new(stats))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -506,10 +502,6 @@ impl ExecutionPlan for AddRowOffsetExec {
         "AddRowOffsetExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -526,7 +518,7 @@ impl ExecutionPlan for AddRowOffsetExec {
         vec![false]
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
diff --git a/rust/lance/src/io/exec/scalar_index.rs b/rust/lance/src/io/exec/scalar_index.rs
index 0f74a13478d..b943ad12d3a 100644
--- a/rust/lance/src/io/exec/scalar_index.rs
+++ b/rust/lance/src/io/exec/scalar_index.rs
@@ -6,7 +6,7 @@ use std::sync::{Arc, LazyLock};
 use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter};
 use crate::{
     Dataset,
-    dataset::rowids::load_row_id_sequences,
+    dataset::rowids::{load_row_id_sequences, translate_addr_treemap_to_row_ids},
     index::{
         prefilter::DatasetPreFilter,
         scalar_logical::{open_named_scalar_index, scalar_index_fragment_bitmap},
@@ -42,7 +42,8 @@ use lance_index::{
     },
 };
 use lance_select::{
-    IndexExprResult, RowAddrMask, RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat,
+    IndexExprResult, NullableIndexExprResult, NullableRowAddrMask, NullableRowAddrSet, RowAddrMask,
+    RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat,
 };
 use lance_table::format::Fragment;
 use roaring::RoaringBitmap;
@@ -58,6 +59,50 @@ impl ScalarIndexLoader for Dataset {
     ) -> Result<Arc<dyn ScalarIndex>> {
         open_named_scalar_index(self, column, index_name, metrics).await
     }
+
+    async fn row_addr_result_to_row_ids(
+        &self,
+        result: NullableIndexExprResult,
+    ) -> Result<NullableIndexExprResult> {
+        // Addresses and row ids only diverge under stable row ids; otherwise the
+        // address is the row id and there is nothing to translate.
+        if !self.manifest.uses_stable_row_ids() {
+            return Ok(result);
+        }
+
+        let NullableIndexExprResult { lower, upper, .. } = result;
+        let lower = translate_addr_mask_to_row_ids(self, lower).await?;
+        let upper = translate_addr_mask_to_row_ids(self, upper).await?;
+        Ok(NullableIndexExprResult::new(lower, upper))
+    }
+}
+
+/// Translate an address-domain [`NullableRowAddrMask`] into the row-id domain
+///
+/// Address-domain index results are always positive allow-lists (`AtMost`), so
+/// a block-list here would mean a boolean op was applied before translation,
+/// which is unsupported.
+async fn translate_addr_mask_to_row_ids(
+    dataset: &Dataset,
+    mask: NullableRowAddrMask,
+) -> Result<NullableRowAddrMask> {
+    match mask {
+        NullableRowAddrMask::AllowList(set) => Ok(NullableRowAddrMask::AllowList(
+            translate_addr_set_to_row_ids(dataset, set).await?,
+        )),
+        NullableRowAddrMask::BlockList(_) => Err(Error::internal(
+            "cannot translate a block-list address mask to the row-id domain",
+        )),
+    }
+}
+
+async fn translate_addr_set_to_row_ids(
+    dataset: &Dataset,
+    set: NullableRowAddrSet,
+) -> Result<NullableRowAddrSet> {
+    let selected = translate_addr_treemap_to_row_ids(dataset, set.selected_rows()).await?;
+    let nulls = translate_addr_treemap_to_row_ids(dataset, set.null_rows()).await?;
+    Ok(NullableRowAddrSet::new(selected, nulls))
 }
 
 /// An execution node that performs a scalar index search
@@ -182,10 +227,6 @@ impl ExecutionPlan for ScalarIndexExec {
         "ScalarIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.result_format.schema().clone()
     }
@@ -233,11 +274,11 @@ impl ExecutionPlan for ScalarIndexExec {
     fn partition_statistics(
         &self,
         _partition: Option<usize>,
-    ) -> datafusion::error::Result<datafusion::physical_plan::Statistics> {
-        Ok(datafusion::physical_plan::Statistics {
+    ) -> datafusion::error::Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(datafusion::physical_plan::Statistics {
             num_rows: datafusion::common::stats::Precision::Exact(2),
             ..datafusion::physical_plan::Statistics::new_unknown(self.result_format.schema())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -489,10 +530,6 @@ impl ExecutionPlan for MapIndexExec {
         "MapIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         INDEX_LOOKUP_SCHEMA.clone()
     }
@@ -561,6 +598,10 @@ pub struct MaterializeIndexExec {
     dataset: Arc<Dataset>,
     expr: ScalarIndexExpr,
     fragments: Arc<Vec<Fragment>>,
+    /// Row addresses blocked from the index result due to data overlay files committed after the
+    /// index was built. ANDead into the candidate mask before row ID materialisation so that stale
+    /// index entries never reach downstream operators.
+    overlay_block: Option<RowAddrMask>,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
 }
@@ -633,16 +674,24 @@ impl MaterializeIndexExec {
             dataset,
             expr,
             fragments,
+            overlay_block: None,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         }
     }
 
+    /// Block specific row addresses (see the `overlay_block` field) from the index result.
+    pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self {
+        self.overlay_block = Some(block);
+        self
+    }
+
     #[instrument(name = "materialize_scalar_index", skip_all, level = "debug")]
     async fn do_execute(
         expr: ScalarIndexExpr,
         dataset: Arc<Dataset>,
         fragments: Arc<Vec<Fragment>>,
+        overlay_block: Option<RowAddrMask>,
         metrics: Arc<IndexMetrics>,
     ) -> Result<RecordBatch> {
         let expr_result = expr.evaluate(dataset.as_ref(), metrics.as_ref());
@@ -670,12 +719,15 @@ impl MaterializeIndexExec {
             }
             Ok(result.upper)
         };
-        let mask = if let Some(prefilter) = prefilter {
+        let mut mask = if let Some(prefilter) = prefilter {
             let (expr_result, prefilter) = futures::try_join!(expr_result, prefilter)?;
             take_upper(expr_result)? & (*prefilter).clone()
         } else {
             take_upper(expr_result.await?)?
         };
+        if let Some(block) = overlay_block {
+            mask = mask & block;
+        }
         let ids = row_ids_for_mask(mask, &dataset, &fragments).await?;
         let ids = UInt64Array::from(ids);
         Ok(RecordBatch::try_new(
@@ -776,10 +828,6 @@ impl ExecutionPlan for MaterializeIndexExec {
         "MaterializeIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         MATERIALIZE_INDEX_SCHEMA.clone()
     }
@@ -811,6 +859,7 @@ impl ExecutionPlan for MaterializeIndexExec {
             self.expr.clone(),
             self.dataset.clone(),
             self.fragments.clone(),
+            self.overlay_block.clone(),
             metrics,
         );
         let stream = futures::stream::iter(vec![batch_fut])
@@ -849,13 +898,15 @@ mod tests {
 
     use crate::index::DatasetIndexExt;
     use arrow::datatypes::UInt64Type;
+    use arrow::record_batch::RecordBatchIterator;
+    use arrow_array::{ArrayRef, Int32Array, RecordBatch};
     use arrow_schema::Schema;
     use datafusion::{
         execution::TaskContext, physical_plan::ExecutionPlan, prelude::SessionConfig,
         scalar::ScalarValue,
     };
     use futures::TryStreamExt;
-    use lance_core::utils::tempfile::TempStrDir;
+    use lance_core::utils::{address::RowAddress, tempfile::TempStrDir};
     use lance_datagen::gen_batch;
     use lance_index::{
         IndexType,
@@ -864,10 +915,11 @@ mod tests {
             expression::{ScalarIndexExpr, ScalarIndexSearch},
         },
     };
-    use lance_select::result::IndexExprResultWireFormat;
+    use lance_select::{RowAddrTreeMap, result::IndexExprResultWireFormat};
 
     use crate::{
         Dataset,
+        dataset::WriteParams,
         io::exec::scalar_index::MaterializeIndexExec,
         utils::test::{DatagenExt, FragmentCount, FragmentRowCount, NoContextTestFixture},
     };
@@ -928,7 +980,6 @@ mod tests {
             needs_recheck: false,
             fragment_bitmap: None,
         });
-
         let fragments = dataset.fragments().clone();
 
         let plan = MaterializeIndexExec::new(dataset, query, fragments);
@@ -949,6 +1000,51 @@ mod tests {
         assert_eq!(batches[0].num_rows(), 5);
     }
 
+    #[tokio::test]
+    async fn test_translate_addr_treemap_to_stable_row_ids() {
+        let test_dir = TempStrDir::default();
+        let batch = RecordBatch::try_from_iter(vec![(
+            "id",
+            Arc::new(Int32Array::from((0..10).collect::<Vec<_>>())) as ArrayRef,
+        )])
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let write_params = WriteParams {
+            enable_stable_row_ids: true,
+            max_rows_per_file: 5,
+            ..Default::default()
+        };
+        let dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+        let fragment_id = dataset.get_fragments()[1].id() as u32;
+
+        let mut full_fragment = RowAddrTreeMap::new();
+        full_fragment.insert_fragment(fragment_id);
+        let translated = super::translate_addr_treemap_to_row_ids(&dataset, &full_fragment)
+            .await
+            .unwrap();
+        let row_ids = translated
+            .get_fragment_bitmap(0)
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>();
+        assert_eq!(row_ids, vec![5, 6, 7, 8, 9]);
+
+        let mut partial_fragment = RowAddrTreeMap::new();
+        partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 1).into());
+        partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 3).into());
+        let translated = super::translate_addr_treemap_to_row_ids(&dataset, &partial_fragment)
+            .await
+            .unwrap();
+        let row_ids = translated
+            .get_fragment_bitmap(0)
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>();
+        assert_eq!(row_ids, vec![6, 8]);
+    }
+
     /// `ScalarIndexExec::schema()` (and the stream it emits) must advertise
     /// the same schema the batch actually carries — otherwise downstream
     /// consumers that trust `ExecutionPlan::schema()` will see a different
diff --git a/rust/lance/src/io/exec/scan.rs b/rust/lance/src/io/exec/scan.rs
index 3ec63ce04cc..a8c3b2e3dc3 100644
--- a/rust/lance/src/io/exec/scan.rs
+++ b/rust/lance/src/io/exec/scan.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::ops::Range;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -26,7 +25,10 @@ use futures::{StreamExt, TryStreamExt};
 use lance_arrow::SchemaExt;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::StreamTracingExt;
-use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD};
+use lance_core::{
+    Error, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID_FIELD,
+    ROW_LAST_UPDATED_AT_VERSION_FIELD,
+};
 use lance_file::reader::FileReaderOptions;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_table::format::Fragment;
@@ -192,7 +194,36 @@ impl LanceStream {
     ) -> Result<Self> {
         let scan_metrics = ScanMetrics::new(metrics, partition);
         let timer = scan_metrics.baseline_metrics.elapsed_compute().timer();
-        let project_schema = projection.clone();
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(projection.as_ref());
+        let read_projection = if materialize_blob_v2_binary {
+            Arc::new(crate::dataset::blob::blob_v2_descriptor_schema(
+                projection.as_ref(),
+            ))
+        } else {
+            projection.clone()
+        };
+        let project_schema = read_projection;
+        let output_projection = if materialize_blob_v2_binary {
+            let mut output_projection = projection.as_ref().clone();
+            let mut system_fields = Vec::with_capacity(4);
+            if config.with_row_id {
+                system_fields.push(ROW_ID_FIELD.clone());
+            }
+            if config.with_row_address {
+                system_fields.push(ROW_ADDR_FIELD.clone());
+            }
+            if config.with_row_last_updated_at_version {
+                system_fields.push(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone());
+            }
+            if config.with_row_created_at_version {
+                system_fields.push(ROW_CREATED_AT_VERSION_FIELD.clone());
+            }
+            output_projection.extend(&system_fields)?;
+            Arc::new(output_projection)
+        } else {
+            projection.clone()
+        };
         let io_parallelism = dataset.object_store.io_parallelism();
         // First, use the value specified by the user in the call
         // Second, use the default from the environment variable, if specified
@@ -275,12 +306,14 @@ impl LanceStream {
 
         let scan_scheduler_clone = scan_scheduler.clone();
 
+        let materialize_dataset = dataset;
         let config_for_stream = config.clone();
         let batches = stream::iter(file_fragments.into_iter().enumerate())
             .map(move |(priority, file_fragment)| {
                 let project_schema = project_schema.clone();
                 let scan_scheduler = scan_scheduler.clone();
                 let config = config_for_stream.clone();
+                let force_row_address = materialize_blob_v2_binary;
                 #[allow(clippy::type_complexity)]
                 let frag_task: BoxFuture<
                     Result<BoxStream<Result<BoxFuture<Result<RecordBatch>>>>>,
@@ -288,7 +321,7 @@ impl LanceStream {
                     (async move {
                         let mut frag_config = FragReadConfig::default()
                             .with_row_id(config.with_row_id)
-                            .with_row_address(config.with_row_address)
+                            .with_row_address(config.with_row_address || force_row_address)
                             .with_row_last_updated_at_version(
                                 config.with_row_last_updated_at_version,
                             )
@@ -349,6 +382,25 @@ impl LanceStream {
             )
             .stream_in_current_span()
             .boxed();
+        let inner_stream = if materialize_blob_v2_binary {
+            inner_stream
+                .and_then(move |batch| {
+                    let dataset = materialize_dataset.clone();
+                    let output_projection = output_projection.clone();
+                    async move {
+                        crate::dataset::blob::materialize_blob_v2_binary_batch(
+                            &dataset,
+                            output_projection.as_ref(),
+                            batch,
+                        )
+                        .await
+                        .map_err(DataFusionError::from)
+                    }
+                })
+                .boxed()
+        } else {
+            inner_stream
+        };
 
         timer.done();
         Ok(Self {
@@ -482,7 +534,9 @@ impl core::fmt::Debug for LanceStream {
 
 impl RecordBatchStream for LanceStream {
     fn schema(&self) -> SchemaRef {
-        let mut schema: ArrowSchema = self.projection.as_ref().into();
+        let output_projection =
+            crate::dataset::blob::public_blob_v2_binary_output_schema(self.projection.as_ref());
+        let mut schema: ArrowSchema = (&output_projection).into();
         if self.config.with_row_id {
             schema = schema.try_with_column(ROW_ID_FIELD.clone()).unwrap();
         }
@@ -602,7 +656,9 @@ impl LanceScanExec {
         projection: Arc<Schema>,
         config: LanceScanConfig,
     ) -> Self {
-        let mut output_schema: ArrowSchema = projection.as_ref().into();
+        let output_projection =
+            crate::dataset::blob::public_blob_v2_binary_output_schema(projection.as_ref());
+        let mut output_schema: ArrowSchema = (&output_projection).into();
 
         if config.with_row_id {
             output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap();
@@ -673,10 +729,6 @@ impl ExecutionPlan for LanceScanExec {
         "LanceScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -727,7 +779,7 @@ impl ExecutionPlan for LanceScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> Result<Arc<Statistics>> {
         // Some fragments from older datasets might have the row count stats missing.
         let (row_count, is_exact) =
             self.fragments
@@ -744,10 +796,10 @@ impl ExecutionPlan for LanceScanExec {
             false => Precision::Absent,
         };
 
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs
index c3642cdb043..fea9e45b3ad 100644
--- a/rust/lance/src/io/exec/take.rs
+++ b/rust/lance/src/io/exec/take.rs
@@ -70,6 +70,10 @@ struct TakeStream {
     dataset: Arc<Dataset>,
     /// The fields to take from the input stream
     fields_to_take: Arc<Schema>,
+    /// The descriptor-view schema used for storage reads when blob payloads
+    /// must be materialized after take.
+    read_fields: Arc<Schema>,
+    materialize_blob_v2_binary: bool,
     /// The output schema, needed for us to merge the new columns
     /// into the input data in the correct order
     output_schema: SchemaRef,
@@ -92,9 +96,20 @@ impl TakeStream {
         metrics: &ExecutionPlanMetricsSet,
         partition: usize,
     ) -> Self {
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(fields_to_take.as_ref());
+        let read_fields = if materialize_blob_v2_binary {
+            Arc::new(crate::dataset::blob::blob_v2_descriptor_schema(
+                fields_to_take.as_ref(),
+            ))
+        } else {
+            fields_to_take.clone()
+        };
         Self {
             dataset,
             fields_to_take,
+            read_fields,
+            materialize_blob_v2_binary,
             output_schema,
             readers_cache: Arc::new(Mutex::new(HashMap::new())),
             scan_scheduler,
@@ -131,14 +146,12 @@ impl TakeStream {
                 ))
             })?;
 
-        let reader = Arc::new(
-            fragment
-                .open(
-                    &self.fields_to_take,
-                    FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()),
-                )
-                .await?,
-        );
+        let mut read_config =
+            FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone());
+        if self.materialize_blob_v2_binary {
+            read_config = read_config.with_row_address(true);
+        }
+        let reader = Arc::new(fragment.open(&self.read_fields, read_config).await?);
 
         let mut readers = self.readers_cache.lock().unwrap();
         readers.insert(fragment_id, reader.clone());
@@ -355,6 +368,15 @@ impl TakeStream {
             (None, None) => {}
         }
 
+        if self.materialize_blob_v2_binary {
+            new_data = crate::dataset::blob::materialize_blob_v2_binary_batch(
+                &self.dataset,
+                self.fields_to_take.as_ref(),
+                new_data,
+            )
+            .await?;
+        }
+
         Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?)
     }
 
@@ -487,10 +509,10 @@ impl TakeExec {
             projection
         );
 
-        let output_schema = Arc::new(Self::calculate_output_schema(
-            dataset.schema(),
-            &input.schema(),
-            &projection,
+        let output_schema =
+            Self::calculate_output_schema(dataset.schema(), &input.schema(), &projection);
+        let output_schema = Arc::new(crate::dataset::blob::public_blob_v2_binary_output_schema(
+            &output_schema,
         ));
         let output_arrow = Arc::new(ArrowSchema::from(output_schema.as_ref()));
         let properties = Arc::new(
@@ -521,7 +543,7 @@ impl TakeExec {
     ///
     /// If this happens the order of the new nested fields will match the order defined in
     /// the dataset schema.
-    fn calculate_output_schema(
+    pub(crate) fn calculate_output_schema(
         dataset_schema: &Schema,
         input_schema: &ArrowSchema,
         projection: &Projection,
@@ -576,10 +598,6 @@ impl ExecutionPlan for TakeExec {
         "TakeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -663,11 +681,11 @@ impl ExecutionPlan for TakeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> Result<datafusion::physical_plan::Statistics> {
-        Ok(Statistics {
+    ) -> Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: self.input.partition_statistics(partition)?.num_rows,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn properties(&self) -> &Arc<PlanProperties> {
diff --git a/rust/lance/src/io/exec/testing.rs b/rust/lance/src/io/exec/testing.rs
index 2d5911a4e46..f979c403431 100644
--- a/rust/lance/src/io/exec/testing.rs
+++ b/rust/lance/src/io/exec/testing.rs
@@ -4,7 +4,6 @@
 //! Testing Node
 //!
 
-use std::any::Any;
 use std::sync::Arc;
 
 use arrow_array::RecordBatch;
@@ -51,10 +50,6 @@ impl ExecutionPlan for TestingExec {
         "TestingExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.batches[0].schema()
     }
diff --git a/rust/lance/src/io/exec/utils.rs b/rust/lance/src/io/exec/utils.rs
index 6e2d50d3736..1af0bc3f4ef 100644
--- a/rust/lance/src/io/exec/utils.rs
+++ b/rust/lance/src/io/exec/utils.rs
@@ -421,10 +421,6 @@ impl ExecutionPlan for ReplayExec {
         "ReplayExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.input.schema()
     }
diff --git a/rust/lance/src/lib.rs b/rust/lance/src/lib.rs
index 2a5d0d2c822..66bc876bd88 100644
--- a/rust/lance/src/lib.rs
+++ b/rust/lance/src/lib.rs
@@ -81,6 +81,8 @@ pub mod datafusion;
 pub mod dataset;
 pub mod index;
 pub mod io;
+#[cfg(feature = "metrics")]
+pub mod metrics;
 pub mod session;
 pub mod table;
 pub mod utils;
diff --git a/rust/lance/src/metrics.md b/rust/lance/src/metrics.md
new file mode 100644
index 00000000000..05d9df0c0e3
--- /dev/null
+++ b/rust/lance/src/metrics.md
@@ -0,0 +1,40 @@
+Lance publishes metrics through the [`metrics`](https://docs.rs/metrics) crate
+facade. Install any recorder (Prometheus, OpenTelemetry, etc.) in your
+application and Lance will emit into it; when no recorder is installed, emission
+is a cheap no-op. Metrics are only emitted when Lance is built with the
+`metrics` feature.
+
+## Object store metrics
+
+These track I/O against the underlying object store. The `base` label
+identifies the store; its cardinality is controlled by the
+`LANCE_OBJECT_STORE_METRICS_LABEL` environment variable:
+
+- `scheme` (default) — the scheme only (`s3`, `gs`, `az`, `file`, `memory`);
+  low, bounded cardinality.
+- `full` — the store's unique prefix (`s3$my-bucket`, `az$container@account`
+  where Azure's account also matters), so multiple buckets on the same cloud
+  can be told apart. Cardinality grows with the number of stores accessed.
+- `off` — omit the `base` label entirely.
+
+`operation` is one of `get`, `put`, `put_part`, `head`, `list`, `delete`,
+`copy`, `rename`, `complete_multipart`, or `abort_multipart`.
+
+Request counts are per logical operation: a `list` or `delete` that spans many
+objects is one request, matching how backends batch them.
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `lance_object_store_requests_total` | counter | `operation`, `base` | Object store requests issued. |
+| `lance_object_store_request_bytes_total` | counter | `operation`, `base` | Bytes transferred by `get`/`put` requests. A `get` is counted once its response body has been fully read. |
+| `lance_object_store_request_duration_seconds` | histogram | `operation`, `base` | Per-request latency, in seconds. For `get` this covers the full body transfer, not just time-to-first-byte. |
+| `lance_object_store_errors_total` | counter | `operation`, `base` | Requests that returned an error. |
+| `lance_object_store_in_flight_requests` | gauge | `operation`, `base` | Requests currently in flight. |
+| `lance_object_store_throttle_total` | counter | `status`, `base` | Throttle responses (HTTP 429 / 503) seen at the HTTP layer, counted per attempt including retries. The `status` label is the numeric HTTP status. |
+| `lance_object_store_retryable_responses_total` | counter | `status`, `base` | Retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, counted per attempt including retries. A superset of `throttle_total`; 409 (conflict) is excluded so commit conflicts are not counted. |
+
+`lance_object_store_throttle_total` and
+`lance_object_store_retryable_responses_total` are recorded only for the native
+cloud stores (S3, GCS, Azure); Opendal-backed stores bypass the HTTP client
+where the counters are installed, so they report the other object store metrics
+but not throttle/retryable counts.
diff --git a/rust/lance/src/metrics.rs b/rust/lance/src/metrics.rs
new file mode 100644
index 00000000000..1d48865e859
--- /dev/null
+++ b/rust/lance/src/metrics.rs
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Metrics published by Lance.
+#![doc = include_str!("metrics.md")]
+//!
+//! The metrics themselves are emitted from the relevant subsystems (for
+//! example object store I/O is instrumented in
+//! [`lance_io::object_store::metrics`]); this module exists to document the
+//! full catalogue of metric names, types, and labels in one place.
diff --git a/rust/lance/src/session/index_caches.rs b/rust/lance/src/session/index_caches.rs
index e93b7208b09..60e52cf55be 100644
--- a/rust/lance/src/session/index_caches.rs
+++ b/rust/lance/src/session/index_caches.rs
@@ -94,16 +94,22 @@ impl CacheKey for FragReuseIndexKey<'_> {
     }
 }
 
-#[derive(Debug)]
-pub struct IndexMetadataKey {
+#[derive(Clone, Copy, Debug)]
+pub struct IndexMetadataKey<'a> {
     pub version: u64,
+    pub store_identity: &'a str,
 }
 
-impl CacheKey for IndexMetadataKey {
+impl CacheKey for IndexMetadataKey<'_> {
     type ValueType = Vec<IndexMetadata>;
 
     fn key(&self) -> Cow<'_, str> {
-        Cow::Owned(self.version.to_string())
+        Cow::Owned(format!(
+            "{}:{}/{}",
+            self.store_identity.len(),
+            self.store_identity,
+            self.version
+        ))
     }
 
     fn type_name() -> &'static str {
@@ -145,3 +151,22 @@ impl CacheKey for ScalarIndexDetailsKey<'_> {
         "ScalarIndexDetails"
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn index_metadata_key_isolates_object_store_identity() {
+        let first = IndexMetadataKey {
+            version: 7,
+            store_identity: "s3$first-options",
+        };
+        let second = IndexMetadataKey {
+            version: 7,
+            store_identity: "s3$second-options",
+        };
+
+        assert_ne!(first.key(), second.key());
+    }
+}
diff --git a/rust/lance/src/utils/test.rs b/rust/lance/src/utils/test.rs
index 3338eee07a8..f804a7cc38a 100644
--- a/rust/lance/src/utils/test.rs
+++ b/rust/lance/src/utils/test.rs
@@ -243,6 +243,7 @@ impl TestDatasetGenerator {
         Fragment {
             id: 0,
             files,
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(batch.num_rows()),
diff --git a/rust/lance/tests/count_pushdown/mod.rs b/rust/lance/tests/count_pushdown/mod.rs
index aaa3f5f539e..d8afa051bca 100644
--- a/rust/lance/tests/count_pushdown/mod.rs
+++ b/rust/lance/tests/count_pushdown/mod.rs
@@ -81,7 +81,7 @@ fn lance_aware_context(dataset: Arc<Dataset>) -> SessionContext {
 fn plan_contains_pushdown(plan: &Arc<dyn ExecutionPlan>) -> bool {
     let mut found = false;
     plan.apply(|node| {
-        if node.as_any().is::<CountFromMaskExec>() {
+        if node.is::<CountFromMaskExec>() {
             found = true;
             Ok(TreeNodeRecursion::Stop)
         } else {
diff --git a/rust/lance/tests/integration_tests.rs b/rust/lance/tests/integration_tests.rs
index 7a6d3e71ca4..e6ea4e321ae 100644
--- a/rust/lance/tests/integration_tests.rs
+++ b/rust/lance/tests/integration_tests.rs
@@ -4,6 +4,7 @@
 // NOTE: we only create one integration test binary, to keep compilation overhead down.
 
 mod count_pushdown;
+mod mem_wal;
 #[cfg(feature = "slow_tests")]
 mod query;
 #[cfg(feature = "slow_tests")]
diff --git a/rust/lance/tests/mem_wal/mod.rs b/rust/lance/tests/mem_wal/mod.rs
new file mode 100644
index 00000000000..c617c2a1589
--- /dev/null
+++ b/rust/lance/tests/mem_wal/mod.rs
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! End-to-end MemWAL tests through the public [`ShardWriter`] surface, as
+//! opposed to the lib-level unit tests that can reach internals directly.
+
+use std::time::Duration;
+
+use arrow_array::cast::AsArray;
+use arrow_array::record_batch;
+use arrow_array::types::Int32Type;
+use lance::dataset::mem_wal::{ShardWriter, ShardWriterConfig};
+use lance_core::FenceReason;
+use lance_io::object_store::ObjectStore;
+use uuid::Uuid;
+
+fn durable_writer_config(shard_id: Uuid) -> ShardWriterConfig {
+    ShardWriterConfig {
+        shard_id,
+        durable_write: true,
+        max_wal_buffer_size: 64 * 1024 * 1024,
+        max_wal_flush_interval: Some(Duration::from_millis(10)),
+        max_memtable_size: 64 * 1024 * 1024,
+        manifest_scan_batch_size: 2,
+        ..Default::default()
+    }
+}
+
+#[tokio::test]
+async fn durable_put_is_readable_through_public_scan() {
+    let base_uri = "memory://".to_string();
+    let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap();
+    let batch = record_batch!(("id", Int32, [1, 2, 3])).unwrap();
+    let writer = ShardWriter::open(
+        store,
+        base_path,
+        base_uri,
+        durable_writer_config(Uuid::new_v4()),
+        batch.schema(),
+        vec![],
+    )
+    .await
+    .unwrap();
+
+    writer.put(vec![batch]).await.unwrap();
+
+    let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap();
+    let scanned_ids = scanned["id"].as_primitive::<Int32Type>();
+    assert_eq!(
+        scanned_ids.values(),
+        &[1, 2, 3],
+        "a durable put must be readable through the public scan API"
+    );
+
+    writer.close().await.unwrap();
+}
+
+/// A durable put into a post-rotation MemTable must be acknowledged only by
+/// its own WAL flush, never by a flush from an older generation.
+///
+/// Batch positions restart at 0 in every MemTable generation. A durability
+/// watch keyed on that generation-local position is satisfied by the previous
+/// generation's flush at the same position — so the first durable put after a
+/// rotation used to return success before its own WAL append completed. Here
+/// a peer writer has claimed a higher epoch in between, so the false success
+/// is observable: the put's own flush is fenced, and the put must surface
+/// that fence instead of reporting durability.
+#[tokio::test]
+async fn durable_put_does_not_alias_across_memtable_generations() {
+    let base_uri = "memory://".to_string();
+    let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap();
+    let shard_id = Uuid::new_v4();
+    let first_batch = record_batch!(("id", Int32, [1])).unwrap();
+    let schema = first_batch.schema();
+    let config = durable_writer_config(shard_id);
+
+    let writer_a = ShardWriter::open(
+        store.clone(),
+        base_path.clone(),
+        base_uri.clone(),
+        config.clone(),
+        schema.clone(),
+        vec![],
+    )
+    .await
+    .unwrap();
+
+    // Write and fully flush generation N, then rotate to generation N+1.
+    writer_a.put(vec![first_batch]).await.unwrap();
+    let first_generation = writer_a.memtable_stats().await.unwrap().generation;
+    writer_a.force_seal_active().await.unwrap();
+    writer_a.wait_for_flush_drain().await.unwrap();
+    let current_generation = writer_a.memtable_stats().await.unwrap().generation;
+    assert_eq!(
+        current_generation,
+        first_generation + 1,
+        "expected rotation from generation {first_generation}, got {current_generation}"
+    );
+
+    // A peer claims a higher epoch, fencing writer A's next WAL append.
+    let writer_b = ShardWriter::open(store, base_path, base_uri, config, schema, vec![])
+        .await
+        .unwrap();
+    assert!(
+        writer_b.epoch() > writer_a.epoch(),
+        "expected peer epoch to increase: writer_a={}, writer_b={}",
+        writer_a.epoch(),
+        writer_b.epoch()
+    );
+
+    // Generation N+1's first durable put lands at the same local batch
+    // position (0..1) that generation N already flushed. It must wait for its
+    // own flush and therefore surface the fence, not ack from the stale
+    // generation.
+    let second_batch = record_batch!(("id", Int32, [2])).unwrap();
+    let error = writer_a
+        .put(vec![second_batch])
+        .await
+        .expect_err("generation-2 durable put must wait for its own WAL flush");
+    assert_eq!(
+        error.fence_reason(),
+        Some(FenceReason::PeerClaimedEpoch),
+        "expected a peer-claimed-epoch fence, got: {error}"
+    );
+    assert!(
+        error.to_string().contains("Writer fenced"),
+        "expected the writer-fenced error prefix, got: {error}"
+    );
+
+    writer_b.close().await.unwrap();
+}
diff --git a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
index 6fcff711a06..531e90b410f 100644
--- a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
+++ b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
@@ -1039,6 +1039,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "chacha20"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.44"
@@ -1124,9 +1135,9 @@ dependencies = [
 
 [[package]]
 name = "cmov"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
 
 [[package]]
 name = "colorchoice"
@@ -2541,11 +2552,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
 dependencies = [
  "cfg-if",
- "js-sys",
  "libc",
  "r-efi 5.3.0",
  "wasip2",
- "wasm-bindgen",
 ]
 
 [[package]]
@@ -2555,10 +2564,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "r-efi 6.0.0",
+ "rand_core 0.10.1",
  "wasip2",
  "wasip3",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -4594,14 +4606,15 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.14"
+version = "0.11.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
 dependencies = [
  "bytes",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.10.2",
+ "rand_pcg",
  "ring",
  "rustc-hash",
  "rustls",
@@ -4675,6 +4688,17 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
+[[package]]
+name = "rand"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.2",
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.3.1"
@@ -4713,6 +4737,12 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
 [[package]]
 name = "rand_distr"
 version = "0.5.1"
@@ -4723,6 +4753,15 @@ dependencies = [
  "rand 0.9.4",
 ]
 
+[[package]]
+name = "rand_pcg"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
+dependencies = [
+ "rand_core 0.10.1",
+]
+
 [[package]]
 name = "rand_xoshiro"
 version = "0.7.0"
diff --git a/test_data/readme.md b/test_data/readme.md
index 8d041a96db5..69f153c22d1 100644
--- a/test_data/readme.md
+++ b/test_data/readme.md
@@ -30,4 +30,9 @@ folder contains a `datagen.py` script that generates one or more lance datasets.
 * `v0.27.1/pq_in_schema`: This dataset uses the old method of storing the PQ
   metadata in the schema metadata in the index file. We switched to storing them
   in a global buffer in https://github.com/lancedb/lance/pull/3829, but still
-  need to be able to read the old format.
\ No newline at end of file
+  need to be able to read the old format.
+* `v3.0.1/fts_v1` and `v4.0.1/fts_v2`: These datasets cover the supported FTS
+  layouts written by stable Lance releases. Each fixture contains posting lists
+  that cross the layout's block boundary and positions used by phrase queries.
+  The v1 fixture also retains the retired `skip_merge` parameter written by Lance
+  3.0.1.
diff --git a/test_data/v3.0.1/datagen.py b/test_data/v3.0.1/datagen.py
new file mode 100644
index 00000000000..2815351490c
--- /dev/null
+++ b/test_data/v3.0.1/datagen.py
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+import os
+import shutil
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+from lance.query import PhraseQuery
+
+EXPECTED_LANCE_VERSION = "3.0.1"
+EXPECTED_FTS_VERSION = 1
+NUM_ROWS = 300
+
+assert lance.__version__ == EXPECTED_LANCE_VERSION
+
+os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION)
+
+dataset_path = Path(__file__).parent / "fts_v1"
+shutil.rmtree(dataset_path, ignore_errors=True)
+
+row_ids = list(range(NUM_ROWS))
+texts = [
+    "lance database compatibility shared"
+    if row_id % 3 == 0
+    else "database lance compatibility shared"
+    for row_id in row_ids
+]
+dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path)
+dataset.create_scalar_index(
+    "text",
+    "INVERTED",
+    with_position=True,
+    skip_merge=True,
+)
+
+index = dataset.describe_indices()[0]
+assert index.segments[0].index_version == EXPECTED_FTS_VERSION
+
+matches = dataset.to_table(full_text_query="compatibility")
+assert set(matches["id"].to_pylist()) == set(row_ids)
+
+phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text"))
+assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3))
diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance
new file mode 100644
index 00000000000..10d6e21f91a
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance differ
diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance
new file mode 100644
index 00000000000..d7314b6b7c2
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance differ
diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance
new file mode 100644
index 00000000000..609d3cbcbb9
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance differ
diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance
new file mode 100644
index 00000000000..614e7e40872
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance differ
diff --git a/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn
new file mode 100644
index 00000000000..2f01cf7c9b2
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn differ
diff --git a/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn
new file mode 100644
index 00000000000..333e934e259
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn differ
diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..d61e99244a2
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest differ
diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..7839e3ac728
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest differ
diff --git a/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance
new file mode 100644
index 00000000000..96e80c7b9a8
Binary files /dev/null and b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance differ
diff --git a/test_data/v4.0.1/datagen.py b/test_data/v4.0.1/datagen.py
new file mode 100644
index 00000000000..accaa57cadf
--- /dev/null
+++ b/test_data/v4.0.1/datagen.py
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+import os
+import shutil
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+from lance.query import PhraseQuery
+
+EXPECTED_LANCE_VERSION = "4.0.1"
+EXPECTED_FTS_VERSION = 2
+NUM_ROWS = 300
+
+assert lance.__version__ == EXPECTED_LANCE_VERSION
+
+os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION)
+
+dataset_path = Path(__file__).parent / "fts_v2"
+shutil.rmtree(dataset_path, ignore_errors=True)
+
+row_ids = list(range(NUM_ROWS))
+texts = [
+    "lance database compatibility shared"
+    if row_id % 3 == 0
+    else "database lance compatibility shared"
+    for row_id in row_ids
+]
+dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path)
+dataset.create_scalar_index("text", "INVERTED", with_position=True)
+
+index = dataset.describe_indices()[0]
+assert index.segments[0].index_version == EXPECTED_FTS_VERSION
+
+matches = dataset.to_table(full_text_query="compatibility")
+assert set(matches["id"].to_pylist()) == set(row_ids)
+
+phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text"))
+assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3))
diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance
new file mode 100644
index 00000000000..1f3d5698f43
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance differ
diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance
new file mode 100644
index 00000000000..d7314b6b7c2
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance differ
diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance
new file mode 100644
index 00000000000..196ad67f15a
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance differ
diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance
new file mode 100644
index 00000000000..614e7e40872
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance differ
diff --git a/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn
new file mode 100644
index 00000000000..f550fa2c193
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn differ
diff --git a/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn
new file mode 100644
index 00000000000..3cbfb8b41cb
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn differ
diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest
new file mode 100644
index 00000000000..6fa1d8bb6e7
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest differ
diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest
new file mode 100644
index 00000000000..cb8ff78ef70
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest differ
diff --git a/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance
new file mode 100644
index 00000000000..96e80c7b9a8
Binary files /dev/null and b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance differ